1/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
2|* *|
3|* "Fast" Instruction Selector for the X86 target *|
4|* *|
5|* Automatically generated file, do not edit! *|
6|* *|
7\*===----------------------------------------------------------------------===*/
8
9
10// FastEmit Immediate Predicate functions.
11static bool Predicate_AndMask64(int64_t Imm) {
12
13 return isMask_64(Imm) && !isUInt<32>(Imm);
14
15}
16static bool Predicate_BTRMask64(int64_t Imm) {
17
18 return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
19
20}
21static bool Predicate_BTCBTSMask64(int64_t Imm) {
22
23 return !isInt<32>(Imm) && isPowerOf2_64(Imm);
24
25}
26static bool Predicate_i16immSExt8(int64_t Imm) {
27 return isInt<8>(Imm);
28}
29static bool Predicate_i32immSExt8(int64_t Imm) {
30 return isInt<8>(Imm);
31}
32static bool Predicate_i64immSExt8(int64_t Imm) {
33 return isInt<8>(Imm);
34}
35static bool Predicate_i64immSExt32(int64_t Imm) {
36 return isInt<32>(Imm);
37}
38
39
40// FastEmit functions for ISD::ABS.
41
42unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
43 if (RetVT.SimpleTy != MVT::v16i8)
44 return 0;
45 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
46 return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
47 }
48 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
49 return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
50 }
51 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
52 return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
53 }
54 return 0;
55}
56
57unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
58 if (RetVT.SimpleTy != MVT::v32i8)
59 return 0;
60 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
61 return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
62 }
63 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
64 return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
65 }
66 return 0;
67}
68
69unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
70 if (RetVT.SimpleTy != MVT::v64i8)
71 return 0;
72 if ((Subtarget->hasBWI())) {
73 return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
74 }
75 return 0;
76}
77
78unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
79 if (RetVT.SimpleTy != MVT::v8i16)
80 return 0;
81 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
82 return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
83 }
84 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
85 return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
86 }
87 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
88 return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
89 }
90 return 0;
91}
92
93unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
94 if (RetVT.SimpleTy != MVT::v16i16)
95 return 0;
96 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
97 return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
98 }
99 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
100 return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
101 }
102 return 0;
103}
104
105unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
106 if (RetVT.SimpleTy != MVT::v32i16)
107 return 0;
108 if ((Subtarget->hasBWI())) {
109 return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
110 }
111 return 0;
112}
113
114unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
115 if (RetVT.SimpleTy != MVT::v4i32)
116 return 0;
117 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
118 return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
119 }
120 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
121 return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
122 }
123 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
124 return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
125 }
126 return 0;
127}
128
129unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
130 if (RetVT.SimpleTy != MVT::v8i32)
131 return 0;
132 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
133 return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
134 }
135 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
136 return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
137 }
138 return 0;
139}
140
141unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
142 if (RetVT.SimpleTy != MVT::v16i32)
143 return 0;
144 if ((Subtarget->hasAVX512())) {
145 return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
146 }
147 return 0;
148}
149
150unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
151 if (RetVT.SimpleTy != MVT::v2i64)
152 return 0;
153 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
154 return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
155 }
156 return 0;
157}
158
159unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
160 if (RetVT.SimpleTy != MVT::v4i64)
161 return 0;
162 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
163 return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
164 }
165 return 0;
166}
167
168unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
169 if (RetVT.SimpleTy != MVT::v8i64)
170 return 0;
171 if ((Subtarget->hasAVX512())) {
172 return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
173 }
174 return 0;
175}
176
177unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
178 switch (VT.SimpleTy) {
179 case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
180 case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
181 case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
182 case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
183 case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
184 case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
185 case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
186 case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
187 case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
188 case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
189 case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
190 case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
191 default: return 0;
192 }
193}
194
195// FastEmit functions for ISD::ANY_EXTEND.
196
197unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
198 if (RetVT.SimpleTy != MVT::i32)
199 return 0;
200 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
201}
202
203unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
204 switch (VT.SimpleTy) {
205 case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
206 default: return 0;
207 }
208}
209
210// FastEmit functions for ISD::BITCAST.
211
212unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
213 if (RetVT.SimpleTy != MVT::f32)
214 return 0;
215 if ((Subtarget->hasAVX512())) {
216 return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
217 }
218 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
219 return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
220 }
221 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
222 return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
223 }
224 return 0;
225}
226
227unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
228 if ((Subtarget->hasAVX512())) {
229 return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
230 }
231 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
232 return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
233 }
234 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
235 return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
236 }
237 return 0;
238}
239
240unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
241 if ((Subtarget->hasMMX())) {
242 return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
243 }
244 return 0;
245}
246
247unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
248switch (RetVT.SimpleTy) {
249 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
250 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
251 default: return 0;
252}
253}
254
255unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
256 if (RetVT.SimpleTy != MVT::i32)
257 return 0;
258 if ((Subtarget->hasAVX512())) {
259 return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
260 }
261 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
262 return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
263 }
264 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
265 return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
266 }
267 return 0;
268}
269
270unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
271 if ((Subtarget->hasAVX512())) {
272 return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
273 }
274 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
275 return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
276 }
277 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
278 return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
279 }
280 return 0;
281}
282
283unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
284 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
285 return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
286 }
287 return 0;
288}
289
290unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
291switch (RetVT.SimpleTy) {
292 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
293 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
294 default: return 0;
295}
296}
297
298unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
299 if ((Subtarget->hasMMX())) {
300 return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
301 }
302 return 0;
303}
304
305unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
306 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
307 return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
308 }
309 return 0;
310}
311
312unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
313switch (RetVT.SimpleTy) {
314 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
315 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
316 default: return 0;
317}
318}
319
320unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
321 switch (VT.SimpleTy) {
322 case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
323 case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
324 case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
325 case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
326 case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
327 default: return 0;
328 }
329}
330
331// FastEmit functions for ISD::BRIND.
332
333unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
334 if (RetVT.SimpleTy != MVT::isVoid)
335 return 0;
336 if ((!Subtarget->is64Bit())) {
337 return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
338 }
339 return 0;
340}
341
342unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
343 if (RetVT.SimpleTy != MVT::isVoid)
344 return 0;
345 if ((!Subtarget->is64Bit())) {
346 return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
347 }
348 return 0;
349}
350
351unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
352 if (RetVT.SimpleTy != MVT::isVoid)
353 return 0;
354 if ((Subtarget->is64Bit())) {
355 return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
356 }
357 return 0;
358}
359
360unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
361 switch (VT.SimpleTy) {
362 case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
363 case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
364 case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
365 default: return 0;
366 }
367}
368
369// FastEmit functions for ISD::BSWAP.
370
371unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
372 if (RetVT.SimpleTy != MVT::i32)
373 return 0;
374 return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
375}
376
377unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
378 if (RetVT.SimpleTy != MVT::i64)
379 return 0;
380 return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
381}
382
383unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
384 switch (VT.SimpleTy) {
385 case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
386 case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
387 default: return 0;
388 }
389}
390
391// FastEmit functions for ISD::CTLZ.
392
393unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
394 if (RetVT.SimpleTy != MVT::i16)
395 return 0;
396 if ((Subtarget->hasLZCNT())) {
397 return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
398 }
399 return 0;
400}
401
402unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
403 if (RetVT.SimpleTy != MVT::i32)
404 return 0;
405 if ((Subtarget->hasLZCNT())) {
406 return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
407 }
408 return 0;
409}
410
411unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
412 if (RetVT.SimpleTy != MVT::i64)
413 return 0;
414 if ((Subtarget->hasLZCNT())) {
415 return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
416 }
417 return 0;
418}
419
420unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
421 if (RetVT.SimpleTy != MVT::v4i32)
422 return 0;
423 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
424 return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
425 }
426 return 0;
427}
428
429unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
430 if (RetVT.SimpleTy != MVT::v8i32)
431 return 0;
432 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
433 return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
434 }
435 return 0;
436}
437
438unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
439 if (RetVT.SimpleTy != MVT::v16i32)
440 return 0;
441 if ((Subtarget->hasCDI())) {
442 return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
443 }
444 return 0;
445}
446
447unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
448 if (RetVT.SimpleTy != MVT::v2i64)
449 return 0;
450 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
451 return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
452 }
453 return 0;
454}
455
456unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
457 if (RetVT.SimpleTy != MVT::v4i64)
458 return 0;
459 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
460 return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
461 }
462 return 0;
463}
464
465unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
466 if (RetVT.SimpleTy != MVT::v8i64)
467 return 0;
468 if ((Subtarget->hasCDI())) {
469 return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
470 }
471 return 0;
472}
473
474unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
475 switch (VT.SimpleTy) {
476 case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
477 case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
478 case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
479 case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
480 case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
481 case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
482 case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
483 case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
484 case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
485 default: return 0;
486 }
487}
488
489// FastEmit functions for ISD::CTPOP.
490
491unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
492 if (RetVT.SimpleTy != MVT::i16)
493 return 0;
494 if ((Subtarget->hasPOPCNT())) {
495 return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
496 }
497 return 0;
498}
499
500unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
501 if (RetVT.SimpleTy != MVT::i32)
502 return 0;
503 if ((Subtarget->hasPOPCNT())) {
504 return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
505 }
506 return 0;
507}
508
509unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
510 if (RetVT.SimpleTy != MVT::i64)
511 return 0;
512 if ((Subtarget->hasPOPCNT())) {
513 return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
514 }
515 return 0;
516}
517
518unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
519 if (RetVT.SimpleTy != MVT::v16i8)
520 return 0;
521 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
522 return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
523 }
524 return 0;
525}
526
527unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
528 if (RetVT.SimpleTy != MVT::v32i8)
529 return 0;
530 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
531 return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
532 }
533 return 0;
534}
535
536unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
537 if (RetVT.SimpleTy != MVT::v64i8)
538 return 0;
539 if ((Subtarget->hasBITALG())) {
540 return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
541 }
542 return 0;
543}
544
545unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
546 if (RetVT.SimpleTy != MVT::v8i16)
547 return 0;
548 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
549 return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
550 }
551 return 0;
552}
553
554unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
555 if (RetVT.SimpleTy != MVT::v16i16)
556 return 0;
557 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
558 return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
559 }
560 return 0;
561}
562
563unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
564 if (RetVT.SimpleTy != MVT::v32i16)
565 return 0;
566 if ((Subtarget->hasBITALG())) {
567 return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
568 }
569 return 0;
570}
571
572unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
573 if (RetVT.SimpleTy != MVT::v4i32)
574 return 0;
575 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
576 return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
577 }
578 return 0;
579}
580
581unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
582 if (RetVT.SimpleTy != MVT::v8i32)
583 return 0;
584 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
585 return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
586 }
587 return 0;
588}
589
590unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
591 if (RetVT.SimpleTy != MVT::v16i32)
592 return 0;
593 if ((Subtarget->hasVPOPCNTDQ())) {
594 return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
595 }
596 return 0;
597}
598
599unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
600 if (RetVT.SimpleTy != MVT::v2i64)
601 return 0;
602 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
603 return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
604 }
605 return 0;
606}
607
608unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
609 if (RetVT.SimpleTy != MVT::v4i64)
610 return 0;
611 if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
612 return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
613 }
614 return 0;
615}
616
617unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
618 if (RetVT.SimpleTy != MVT::v8i64)
619 return 0;
620 if ((Subtarget->hasVPOPCNTDQ())) {
621 return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
622 }
623 return 0;
624}
625
626unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
627 switch (VT.SimpleTy) {
628 case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
629 case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
630 case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
631 case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
632 case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
633 case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
634 case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
635 case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
636 case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
637 case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
638 case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
639 case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
640 case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
641 case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
642 case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
643 default: return 0;
644 }
645}
646
647// FastEmit functions for ISD::CTTZ.
648
649unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
650 if (RetVT.SimpleTy != MVT::i16)
651 return 0;
652 if ((Subtarget->hasBMI())) {
653 return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
654 }
655 return 0;
656}
657
658unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
659 if (RetVT.SimpleTy != MVT::i32)
660 return 0;
661 if ((Subtarget->hasBMI())) {
662 return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
663 }
664 return 0;
665}
666
667unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
668 if (RetVT.SimpleTy != MVT::i64)
669 return 0;
670 if ((Subtarget->hasBMI())) {
671 return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
672 }
673 return 0;
674}
675
676unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
677 switch (VT.SimpleTy) {
678 case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
679 case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
680 case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
681 default: return 0;
682 }
683}
684
685// FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
686
687unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
688 if (RetVT.SimpleTy != MVT::i16)
689 return 0;
690 return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
691}
692
693unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
694 if (RetVT.SimpleTy != MVT::i32)
695 return 0;
696 return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
697}
698
699unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
700 if (RetVT.SimpleTy != MVT::i64)
701 return 0;
702 return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
703}
704
705unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
706 switch (VT.SimpleTy) {
707 case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
708 case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
709 case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
710 default: return 0;
711 }
712}
713
714// FastEmit functions for ISD::FABS.
715
716unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
717 if (RetVT.SimpleTy != MVT::f32)
718 return 0;
719 if ((!Subtarget->hasSSE1())) {
720 return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
721 }
722 return 0;
723}
724
725unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
726 if (RetVT.SimpleTy != MVT::f64)
727 return 0;
728 if ((!Subtarget->hasSSE2())) {
729 return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
730 }
731 return 0;
732}
733
734unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
735 if (RetVT.SimpleTy != MVT::f80)
736 return 0;
737 return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
738}
739
740unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
741 switch (VT.SimpleTy) {
742 case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
743 case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
744 case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
745 default: return 0;
746 }
747}
748
749// FastEmit functions for ISD::FNEG.
750
751unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
752 if (RetVT.SimpleTy != MVT::f32)
753 return 0;
754 if ((!Subtarget->hasSSE1())) {
755 return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
756 }
757 return 0;
758}
759
760unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
761 if (RetVT.SimpleTy != MVT::f64)
762 return 0;
763 if ((!Subtarget->hasSSE2())) {
764 return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
765 }
766 return 0;
767}
768
769unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
770 if (RetVT.SimpleTy != MVT::f80)
771 return 0;
772 return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
773}
774
775unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
776 switch (VT.SimpleTy) {
777 case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
778 case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
779 case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
780 default: return 0;
781 }
782}
783
784// FastEmit functions for ISD::FP_EXTEND.
785
786unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
787 if (RetVT.SimpleTy != MVT::f64)
788 return 0;
789 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
790 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
791 }
792 return 0;
793}
794
795unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
796 if (RetVT.SimpleTy != MVT::v4f64)
797 return 0;
798 if ((Subtarget->hasVLX())) {
799 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
800 }
801 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
802 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
803 }
804 return 0;
805}
806
807unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
808 if (RetVT.SimpleTy != MVT::v8f64)
809 return 0;
810 if ((Subtarget->hasAVX512())) {
811 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
812 }
813 return 0;
814}
815
816unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
817 switch (VT.SimpleTy) {
818 case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
819 case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
820 case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
821 default: return 0;
822 }
823}
824
825// FastEmit functions for ISD::FP_ROUND.
826
827unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
828 if (RetVT.SimpleTy != MVT::f32)
829 return 0;
830 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
831 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
832 }
833 return 0;
834}
835
836unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
837 switch (VT.SimpleTy) {
838 case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
839 default: return 0;
840 }
841}
842
843// FastEmit functions for ISD::FP_TO_SINT.
844
845unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
846 if ((Subtarget->hasAVX512())) {
847 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
848 }
849 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
850 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
851 }
852 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
853 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
854 }
855 return 0;
856}
857
858unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
859 if ((Subtarget->hasAVX512())) {
860 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
861 }
862 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
863 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
864 }
865 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
866 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
867 }
868 return 0;
869}
870
871unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
872switch (RetVT.SimpleTy) {
873 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
874 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
875 default: return 0;
876}
877}
878
879unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
880 if ((Subtarget->hasAVX512())) {
881 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
882 }
883 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
884 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
885 }
886 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
887 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
888 }
889 return 0;
890}
891
892unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
893 if ((Subtarget->hasAVX512())) {
894 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
895 }
896 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
897 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
898 }
899 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
900 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
901 }
902 return 0;
903}
904
905unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
906switch (RetVT.SimpleTy) {
907 case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
908 case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
909 default: return 0;
910}
911}
912
913unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
914 if (RetVT.SimpleTy != MVT::v4i32)
915 return 0;
916 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
917 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
918 }
919 return 0;
920}
921
922unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
923 switch (VT.SimpleTy) {
924 case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
925 case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
926 case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
927 default: return 0;
928 }
929}
930
931// FastEmit functions for ISD::FP_TO_UINT.
932
933unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
934 if ((Subtarget->hasAVX512())) {
935 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
936 }
937 return 0;
938}
939
940unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
941 if ((Subtarget->hasAVX512())) {
942 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
943 }
944 return 0;
945}
946
947unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
948switch (RetVT.SimpleTy) {
949 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
950 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
951 default: return 0;
952}
953}
954
955unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
956 if ((Subtarget->hasAVX512())) {
957 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
958 }
959 return 0;
960}
961
962unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
963 if ((Subtarget->hasAVX512())) {
964 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
965 }
966 return 0;
967}
968
969unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
970switch (RetVT.SimpleTy) {
971 case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
972 case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
973 default: return 0;
974}
975}
976
977unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
978 switch (VT.SimpleTy) {
979 case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
980 case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
981 default: return 0;
982 }
983}
984
985// FastEmit functions for ISD::FSQRT.
986
987unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
988 if (RetVT.SimpleTy != MVT::f32)
989 return 0;
990 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
991 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
992 }
993 if ((!Subtarget->hasSSE1())) {
994 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
995 }
996 return 0;
997}
998
999unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1000 if (RetVT.SimpleTy != MVT::f64)
1001 return 0;
1002 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1003 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
1004 }
1005 if ((!Subtarget->hasSSE2())) {
1006 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
1007 }
1008 return 0;
1009}
1010
1011unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1012 if (RetVT.SimpleTy != MVT::f80)
1013 return 0;
1014 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
1015}
1016
1017unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1018 if (RetVT.SimpleTy != MVT::v4f32)
1019 return 0;
1020 if ((Subtarget->hasVLX())) {
1021 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1022 }
1023 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1024 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1025 }
1026 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1027 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
1028 }
1029 return 0;
1030}
1031
1032unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1033 if (RetVT.SimpleTy != MVT::v8f32)
1034 return 0;
1035 if ((Subtarget->hasVLX())) {
1036 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1037 }
1038 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1039 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
1040 }
1041 return 0;
1042}
1043
1044unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1045 if (RetVT.SimpleTy != MVT::v16f32)
1046 return 0;
1047 if ((Subtarget->hasAVX512())) {
1048 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
1049 }
1050 return 0;
1051}
1052
1053unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1054 if (RetVT.SimpleTy != MVT::v2f64)
1055 return 0;
1056 if ((Subtarget->hasVLX())) {
1057 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
1058 }
1059 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1060 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1061 }
1062 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1063 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
1064 }
1065 return 0;
1066}
1067
1068unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1069 if (RetVT.SimpleTy != MVT::v4f64)
1070 return 0;
1071 if ((Subtarget->hasVLX())) {
1072 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
1073 }
1074 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1075 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
1076 }
1077 return 0;
1078}
1079
1080unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1081 if (RetVT.SimpleTy != MVT::v8f64)
1082 return 0;
1083 if ((Subtarget->hasAVX512())) {
1084 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
1085 }
1086 return 0;
1087}
1088
1089unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1090 switch (VT.SimpleTy) {
1091 case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1092 case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1093 case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
1094 case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1095 case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1096 case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
1097 case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
1098 case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
1099 case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
1100 default: return 0;
1101 }
1102}
1103
1104// FastEmit functions for ISD::LLRINT.
1105
1106unsigned fastEmit_ISD_LLRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1107 if (RetVT.SimpleTy != MVT::i64)
1108 return 0;
1109 if ((Subtarget->hasAVX512())) {
1110 return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1111 }
1112 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1113 return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1114 }
1115 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1116 return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1117 }
1118 return 0;
1119}
1120
1121unsigned fastEmit_ISD_LLRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1122 if (RetVT.SimpleTy != MVT::i64)
1123 return 0;
1124 if ((Subtarget->hasAVX512())) {
1125 return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1126 }
1127 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1128 return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1129 }
1130 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1131 return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1132 }
1133 return 0;
1134}
1135
1136unsigned fastEmit_ISD_LLRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1137 switch (VT.SimpleTy) {
1138 case MVT::f32: return fastEmit_ISD_LLRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1139 case MVT::f64: return fastEmit_ISD_LLRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1140 default: return 0;
1141 }
1142}
1143
1144// FastEmit functions for ISD::LRINT.
1145
1146unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1147 if ((Subtarget->hasAVX512())) {
1148 return fastEmitInst_r(X86::VCVTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1149 }
1150 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1151 return fastEmitInst_r(X86::CVTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1152 }
1153 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1154 return fastEmitInst_r(X86::VCVTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1155 }
1156 return 0;
1157}
1158
1159unsigned fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1160 if ((Subtarget->hasAVX512())) {
1161 return fastEmitInst_r(X86::VCVTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1162 }
1163 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1164 return fastEmitInst_r(X86::CVTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1165 }
1166 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1167 return fastEmitInst_r(X86::VCVTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1168 }
1169 return 0;
1170}
1171
1172unsigned fastEmit_ISD_LRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1173switch (RetVT.SimpleTy) {
1174 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
1175 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
1176 default: return 0;
1177}
1178}
1179
1180unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1181 if ((Subtarget->hasAVX512())) {
1182 return fastEmitInst_r(X86::VCVTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1183 }
1184 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1185 return fastEmitInst_r(X86::CVTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1186 }
1187 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1188 return fastEmitInst_r(X86::VCVTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1189 }
1190 return 0;
1191}
1192
1193unsigned fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1194 if ((Subtarget->hasAVX512())) {
1195 return fastEmitInst_r(X86::VCVTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1196 }
1197 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1198 return fastEmitInst_r(X86::CVTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1199 }
1200 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1201 return fastEmitInst_r(X86::VCVTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1202 }
1203 return 0;
1204}
1205
1206unsigned fastEmit_ISD_LRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1207switch (RetVT.SimpleTy) {
1208 case MVT::i32: return fastEmit_ISD_LRINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
1209 case MVT::i64: return fastEmit_ISD_LRINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
1210 default: return 0;
1211}
1212}
1213
1214unsigned fastEmit_ISD_LRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1215 switch (VT.SimpleTy) {
1216 case MVT::f32: return fastEmit_ISD_LRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
1217 case MVT::f64: return fastEmit_ISD_LRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
1218 default: return 0;
1219 }
1220}
1221
1222// FastEmit functions for ISD::SCALAR_TO_VECTOR.
1223
1224unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1225 if (RetVT.SimpleTy != MVT::v4i32)
1226 return 0;
1227 if ((Subtarget->hasAVX512())) {
1228 return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1229 }
1230 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1231 return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1232 }
1233 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1234 return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1235 }
1236 return 0;
1237}
1238
1239unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1240 if (RetVT.SimpleTy != MVT::v2i64)
1241 return 0;
1242 if ((Subtarget->hasAVX512())) {
1243 return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
1244 }
1245 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1246 return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1247 }
1248 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1249 return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
1250 }
1251 return 0;
1252}
1253
1254unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1255 switch (VT.SimpleTy) {
1256 case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
1257 case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
1258 default: return 0;
1259 }
1260}
1261
1262// FastEmit functions for ISD::SIGN_EXTEND.
1263
1264unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1265 return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
1266}
1267
1268unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1269 return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
1270}
1271
1272unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1273switch (RetVT.SimpleTy) {
1274 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
1275 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
1276 default: return 0;
1277}
1278}
1279
1280unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1281 return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
1282}
1283
1284unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1285 return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
1286}
1287
1288unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1289switch (RetVT.SimpleTy) {
1290 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
1291 case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
1292 default: return 0;
1293}
1294}
1295
1296unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1297 if (RetVT.SimpleTy != MVT::i64)
1298 return 0;
1299 if ((Subtarget->is64Bit())) {
1300 return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
1301 }
1302 return 0;
1303}
1304
1305unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1306 if (RetVT.SimpleTy != MVT::v2i64)
1307 return 0;
1308 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1309 return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1310 }
1311 return 0;
1312}
1313
1314unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1315 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1316 return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1317 }
1318 return 0;
1319}
1320
1321unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1322 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1323 return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1324 }
1325 return 0;
1326}
1327
1328unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1329switch (RetVT.SimpleTy) {
1330 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
1331 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
1332 default: return 0;
1333}
1334}
1335
1336unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1337 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1338 return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1339 }
1340 return 0;
1341}
1342
1343unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1344 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1345 return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1346 }
1347 return 0;
1348}
1349
1350unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1351 if ((Subtarget->hasDQI())) {
1352 return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1353 }
1354 return 0;
1355}
1356
1357unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1358switch (RetVT.SimpleTy) {
1359 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
1360 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
1361 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
1362 default: return 0;
1363}
1364}
1365
1366unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
1367 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1368 return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1369 }
1370 return 0;
1371}
1372
1373unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
1374 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1375 return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1376 }
1377 return 0;
1378}
1379
1380unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
1381 if ((Subtarget->hasDQI())) {
1382 return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1383 }
1384 return 0;
1385}
1386
1387unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1388switch (RetVT.SimpleTy) {
1389 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
1390 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
1391 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
1392 default: return 0;
1393}
1394}
1395
1396unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
1397 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1398 return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1399 }
1400 return 0;
1401}
1402
1403unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
1404 if ((Subtarget->hasBWI())) {
1405 return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1406 }
1407 return 0;
1408}
1409
1410unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1411switch (RetVT.SimpleTy) {
1412 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
1413 case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
1414 default: return 0;
1415}
1416}
1417
1418unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1419 if (RetVT.SimpleTy != MVT::v64i8)
1420 return 0;
1421 if ((Subtarget->hasBWI())) {
1422 return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1423 }
1424 return 0;
1425}
1426
1427unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
1428 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1429 return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1430 }
1431 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1432 return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1433 }
1434 return 0;
1435}
1436
1437unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
1438 if ((Subtarget->hasAVX512())) {
1439 return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1440 }
1441 return 0;
1442}
1443
1444unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1445switch (RetVT.SimpleTy) {
1446 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
1447 case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
1448 default: return 0;
1449}
1450}
1451
1452unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1453 if (RetVT.SimpleTy != MVT::v32i16)
1454 return 0;
1455 if ((Subtarget->hasBWI())) {
1456 return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1457 }
1458 return 0;
1459}
1460
1461unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1462 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1463 return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1464 }
1465 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1466 return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1467 }
1468 return 0;
1469}
1470
1471unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1472 if ((Subtarget->hasAVX512())) {
1473 return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1474 }
1475 return 0;
1476}
1477
1478unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1479switch (RetVT.SimpleTy) {
1480 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
1481 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
1482 default: return 0;
1483}
1484}
1485
1486unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1487 if (RetVT.SimpleTy != MVT::v16i32)
1488 return 0;
1489 if ((Subtarget->hasAVX512())) {
1490 return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1491 }
1492 return 0;
1493}
1494
1495unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1496 if (RetVT.SimpleTy != MVT::v4i64)
1497 return 0;
1498 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1499 return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1500 }
1501 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1502 return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1503 }
1504 return 0;
1505}
1506
1507unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1508 if (RetVT.SimpleTy != MVT::v8i64)
1509 return 0;
1510 if ((Subtarget->hasAVX512())) {
1511 return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1512 }
1513 return 0;
1514}
1515
1516unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1517 switch (VT.SimpleTy) {
1518 case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
1519 case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
1520 case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
1521 case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
1522 case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
1523 case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
1524 case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
1525 case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
1526 case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
1527 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1528 case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
1529 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1530 case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
1531 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1532 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
1533 default: return 0;
1534 }
1535}
1536
1537// FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
1538
1539unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
1540 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1541 return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1542 }
1543 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
1544 return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
1545 }
1546 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
1547 return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1548 }
1549 return 0;
1550}
1551
1552unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1553 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1554 return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1555 }
1556 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1557 return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1558 }
1559 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1560 return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1561 }
1562 return 0;
1563}
1564
1565unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
1566 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1567 return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1568 }
1569 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1570 return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1571 }
1572 return 0;
1573}
1574
1575unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1576 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1577 return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1578 }
1579 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1580 return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1581 }
1582 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1583 return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1584 }
1585 return 0;
1586}
1587
1588unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1589 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1590 return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1591 }
1592 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1593 return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1594 }
1595 return 0;
1596}
1597
1598unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
1599 if ((Subtarget->hasAVX512())) {
1600 return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1601 }
1602 return 0;
1603}
1604
1605unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1606switch (RetVT.SimpleTy) {
1607 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
1608 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
1609 case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
1610 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
1611 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
1612 case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
1613 default: return 0;
1614}
1615}
1616
1617unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
1618 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1619 return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1620 }
1621 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1622 return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
1623 }
1624 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1625 return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1626 }
1627 return 0;
1628}
1629
1630unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
1631 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1632 return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1633 }
1634 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1635 return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1636 }
1637 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1638 return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1639 }
1640 return 0;
1641}
1642
1643unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
1644 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
1645 return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1646 }
1647 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1648 return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1649 }
1650 return 0;
1651}
1652
1653unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1654switch (RetVT.SimpleTy) {
1655 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
1656 case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
1657 case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
1658 default: return 0;
1659}
1660}
1661
1662unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1663 if (RetVT.SimpleTy != MVT::v2i64)
1664 return 0;
1665 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
1666 return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1667 }
1668 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1669 return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
1670 }
1671 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
1672 return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1673 }
1674 return 0;
1675}
1676
1677unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1678 switch (VT.SimpleTy) {
1679 case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
1680 case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
1681 case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1682 default: return 0;
1683 }
1684}
1685
1686// FastEmit functions for ISD::SINT_TO_FP.
1687
1688unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1689 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1690 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1691 }
1692 return 0;
1693}
1694
1695unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1696 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1697 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1698 }
1699 return 0;
1700}
1701
1702unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1703switch (RetVT.SimpleTy) {
1704 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
1705 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
1706 default: return 0;
1707}
1708}
1709
1710unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
1711 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1712 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1713 }
1714 return 0;
1715}
1716
1717unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
1718 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1719 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1720 }
1721 return 0;
1722}
1723
1724unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1725switch (RetVT.SimpleTy) {
1726 case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
1727 case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
1728 default: return 0;
1729}
1730}
1731
1732unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1733 if ((Subtarget->hasVLX())) {
1734 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1735 }
1736 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1737 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1738 }
1739 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1740 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
1741 }
1742 return 0;
1743}
1744
1745unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1746 if ((Subtarget->hasVLX())) {
1747 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1748 }
1749 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1750 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1751 }
1752 return 0;
1753}
1754
1755unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1756switch (RetVT.SimpleTy) {
1757 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
1758 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
1759 default: return 0;
1760}
1761}
1762
1763unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1764 if ((Subtarget->hasVLX())) {
1765 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1766 }
1767 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1768 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1769 }
1770 return 0;
1771}
1772
1773unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1774 if ((Subtarget->hasAVX512())) {
1775 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1776 }
1777 return 0;
1778}
1779
1780unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1781switch (RetVT.SimpleTy) {
1782 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
1783 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
1784 default: return 0;
1785}
1786}
1787
1788unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1789 if (RetVT.SimpleTy != MVT::v16f32)
1790 return 0;
1791 if ((Subtarget->hasAVX512())) {
1792 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1793 }
1794 return 0;
1795}
1796
1797unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1798 if (RetVT.SimpleTy != MVT::v2f64)
1799 return 0;
1800 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1801 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1802 }
1803 return 0;
1804}
1805
1806unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
1807 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1808 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
1809 }
1810 return 0;
1811}
1812
1813unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
1814 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
1815 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1816 }
1817 return 0;
1818}
1819
1820unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1821switch (RetVT.SimpleTy) {
1822 case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
1823 case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
1824 default: return 0;
1825}
1826}
1827
1828unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
1829 if ((Subtarget->hasDQI())) {
1830 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
1831 }
1832 return 0;
1833}
1834
1835unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
1836 if ((Subtarget->hasDQI())) {
1837 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1838 }
1839 return 0;
1840}
1841
1842unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1843switch (RetVT.SimpleTy) {
1844 case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
1845 case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
1846 default: return 0;
1847}
1848}
1849
1850unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1851 switch (VT.SimpleTy) {
1852 case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
1853 case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
1854 case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
1855 case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
1856 case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
1857 case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
1858 case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
1859 case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
1860 default: return 0;
1861 }
1862}
1863
1864// FastEmit functions for ISD::STRICT_FP_EXTEND.
1865
1866unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1867 if (RetVT.SimpleTy != MVT::f64)
1868 return 0;
1869 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1870 return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
1871 }
1872 return 0;
1873}
1874
1875unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1876 if (RetVT.SimpleTy != MVT::v4f64)
1877 return 0;
1878 if ((Subtarget->hasVLX())) {
1879 return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
1880 }
1881 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1882 return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
1883 }
1884 return 0;
1885}
1886
1887unsigned fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1888 if (RetVT.SimpleTy != MVT::v8f64)
1889 return 0;
1890 if ((Subtarget->hasAVX512())) {
1891 return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
1892 }
1893 return 0;
1894}
1895
1896unsigned fastEmit_ISD_STRICT_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1897 switch (VT.SimpleTy) {
1898 case MVT::f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
1899 case MVT::v4f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
1900 case MVT::v8f32: return fastEmit_ISD_STRICT_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
1901 default: return 0;
1902 }
1903}
1904
1905// FastEmit functions for ISD::STRICT_FP_ROUND.
1906
1907unsigned fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1908 if (RetVT.SimpleTy != MVT::f32)
1909 return 0;
1910 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1911 return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
1912 }
1913 return 0;
1914}
1915
1916unsigned fastEmit_ISD_STRICT_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
1917 switch (VT.SimpleTy) {
1918 case MVT::f64: return fastEmit_ISD_STRICT_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
1919 default: return 0;
1920 }
1921}
1922
1923// FastEmit functions for ISD::STRICT_FP_TO_SINT.
1924
1925unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1926 if ((Subtarget->hasAVX512())) {
1927 return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1928 }
1929 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1930 return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1931 }
1932 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1933 return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1934 }
1935 return 0;
1936}
1937
1938unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1939 if ((Subtarget->hasAVX512())) {
1940 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1941 }
1942 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
1943 return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1944 }
1945 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1946 return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1947 }
1948 return 0;
1949}
1950
1951unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1952switch (RetVT.SimpleTy) {
1953 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
1954 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
1955 default: return 0;
1956}
1957}
1958
1959unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
1960 if ((Subtarget->hasAVX512())) {
1961 return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
1962 }
1963 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1964 return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1965 }
1966 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1967 return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
1968 }
1969 return 0;
1970}
1971
1972unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
1973 if ((Subtarget->hasAVX512())) {
1974 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
1975 }
1976 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
1977 return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1978 }
1979 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
1980 return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
1981 }
1982 return 0;
1983}
1984
1985unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1986switch (RetVT.SimpleTy) {
1987 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
1988 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
1989 default: return 0;
1990}
1991}
1992
1993unsigned fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
1994 if (RetVT.SimpleTy != MVT::v4i32)
1995 return 0;
1996 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
1997 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
1998 }
1999 return 0;
2000}
2001
2002unsigned fastEmit_ISD_STRICT_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2003 switch (VT.SimpleTy) {
2004 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2005 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2006 case MVT::v4f64: return fastEmit_ISD_STRICT_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2007 default: return 0;
2008 }
2009}
2010
2011// FastEmit functions for ISD::STRICT_FP_TO_UINT.
2012
2013unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
2014 if ((Subtarget->hasAVX512())) {
2015 return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
2016 }
2017 return 0;
2018}
2019
2020unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
2021 if ((Subtarget->hasAVX512())) {
2022 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
2023 }
2024 return 0;
2025}
2026
2027unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2028switch (RetVT.SimpleTy) {
2029 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
2030 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
2031 default: return 0;
2032}
2033}
2034
2035unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
2036 if ((Subtarget->hasAVX512())) {
2037 return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
2038 }
2039 return 0;
2040}
2041
2042unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
2043 if ((Subtarget->hasAVX512())) {
2044 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
2045 }
2046 return 0;
2047}
2048
2049unsigned fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2050switch (RetVT.SimpleTy) {
2051 case MVT::i32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
2052 case MVT::i64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
2053 default: return 0;
2054}
2055}
2056
2057unsigned fastEmit_ISD_STRICT_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2058 switch (VT.SimpleTy) {
2059 case MVT::f32: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2060 case MVT::f64: return fastEmit_ISD_STRICT_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2061 default: return 0;
2062 }
2063}
2064
2065// FastEmit functions for ISD::STRICT_FSQRT.
2066
2067unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2068 if (RetVT.SimpleTy != MVT::f32)
2069 return 0;
2070 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2071 return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
2072 }
2073 if ((!Subtarget->hasSSE1())) {
2074 return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
2075 }
2076 return 0;
2077}
2078
2079unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2080 if (RetVT.SimpleTy != MVT::f64)
2081 return 0;
2082 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2083 return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
2084 }
2085 if ((!Subtarget->hasSSE2())) {
2086 return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
2087 }
2088 return 0;
2089}
2090
2091unsigned fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2092 if (RetVT.SimpleTy != MVT::f80)
2093 return 0;
2094 return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
2095}
2096
2097unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2098 if (RetVT.SimpleTy != MVT::v4f32)
2099 return 0;
2100 if ((Subtarget->hasVLX())) {
2101 return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
2102 }
2103 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2104 return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
2105 }
2106 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2107 return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
2108 }
2109 return 0;
2110}
2111
2112unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2113 if (RetVT.SimpleTy != MVT::v8f32)
2114 return 0;
2115 if ((Subtarget->hasVLX())) {
2116 return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
2117 }
2118 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2119 return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
2120 }
2121 return 0;
2122}
2123
2124unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2125 if (RetVT.SimpleTy != MVT::v16f32)
2126 return 0;
2127 if ((Subtarget->hasAVX512())) {
2128 return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
2129 }
2130 return 0;
2131}
2132
2133unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2134 if (RetVT.SimpleTy != MVT::v2f64)
2135 return 0;
2136 if ((Subtarget->hasVLX())) {
2137 return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
2138 }
2139 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2140 return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
2141 }
2142 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2143 return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
2144 }
2145 return 0;
2146}
2147
2148unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2149 if (RetVT.SimpleTy != MVT::v4f64)
2150 return 0;
2151 if ((Subtarget->hasVLX())) {
2152 return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
2153 }
2154 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2155 return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
2156 }
2157 return 0;
2158}
2159
2160unsigned fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2161 if (RetVT.SimpleTy != MVT::v8f64)
2162 return 0;
2163 if ((Subtarget->hasAVX512())) {
2164 return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
2165 }
2166 return 0;
2167}
2168
2169unsigned fastEmit_ISD_STRICT_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2170 switch (VT.SimpleTy) {
2171 case MVT::f32: return fastEmit_ISD_STRICT_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
2172 case MVT::f64: return fastEmit_ISD_STRICT_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
2173 case MVT::f80: return fastEmit_ISD_STRICT_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
2174 case MVT::v4f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
2175 case MVT::v8f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
2176 case MVT::v16f32: return fastEmit_ISD_STRICT_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
2177 case MVT::v2f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
2178 case MVT::v4f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
2179 case MVT::v8f64: return fastEmit_ISD_STRICT_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
2180 default: return 0;
2181 }
2182}
2183
2184// FastEmit functions for ISD::STRICT_SINT_TO_FP.
2185
2186unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
2187 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2188 return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
2189 }
2190 return 0;
2191}
2192
2193unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2194 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2195 return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
2196 }
2197 return 0;
2198}
2199
2200unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2201switch (RetVT.SimpleTy) {
2202 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
2203 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
2204 default: return 0;
2205}
2206}
2207
2208unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
2209 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
2210 return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
2211 }
2212 return 0;
2213}
2214
2215unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
2216 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2217 return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
2218 }
2219 return 0;
2220}
2221
2222unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2223switch (RetVT.SimpleTy) {
2224 case MVT::f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
2225 case MVT::f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
2226 default: return 0;
2227}
2228}
2229
2230unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2231 if ((Subtarget->hasVLX())) {
2232 return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2233 }
2234 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
2235 return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
2236 }
2237 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2238 return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
2239 }
2240 return 0;
2241}
2242
2243unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2244 if ((Subtarget->hasVLX())) {
2245 return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2246 }
2247 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2248 return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2249 }
2250 return 0;
2251}
2252
2253unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2254switch (RetVT.SimpleTy) {
2255 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
2256 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
2257 default: return 0;
2258}
2259}
2260
2261unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2262 if ((Subtarget->hasVLX())) {
2263 return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2264 }
2265 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2266 return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2267 }
2268 return 0;
2269}
2270
2271unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2272 if ((Subtarget->hasAVX512())) {
2273 return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2274 }
2275 return 0;
2276}
2277
2278unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2279switch (RetVT.SimpleTy) {
2280 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
2281 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
2282 default: return 0;
2283}
2284}
2285
2286unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2287 if (RetVT.SimpleTy != MVT::v16f32)
2288 return 0;
2289 if ((Subtarget->hasAVX512())) {
2290 return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2291 }
2292 return 0;
2293}
2294
2295unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2296 if (RetVT.SimpleTy != MVT::v2f64)
2297 return 0;
2298 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2299 return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2300 }
2301 return 0;
2302}
2303
2304unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2305 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2306 return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2307 }
2308 return 0;
2309}
2310
2311unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2312 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2313 return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2314 }
2315 return 0;
2316}
2317
2318unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2319switch (RetVT.SimpleTy) {
2320 case MVT::v4f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
2321 case MVT::v4f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
2322 default: return 0;
2323}
2324}
2325
2326unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2327 if ((Subtarget->hasDQI())) {
2328 return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2329 }
2330 return 0;
2331}
2332
2333unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2334 if ((Subtarget->hasDQI())) {
2335 return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2336 }
2337 return 0;
2338}
2339
2340unsigned fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2341switch (RetVT.SimpleTy) {
2342 case MVT::v8f32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
2343 case MVT::v8f64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
2344 default: return 0;
2345}
2346}
2347
2348unsigned fastEmit_ISD_STRICT_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2349 switch (VT.SimpleTy) {
2350 case MVT::i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
2351 case MVT::i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
2352 case MVT::v4i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2353 case MVT::v8i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2354 case MVT::v16i32: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2355 case MVT::v2i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2356 case MVT::v4i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2357 case MVT::v8i64: return fastEmit_ISD_STRICT_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2358 default: return 0;
2359 }
2360}
2361
2362// FastEmit functions for ISD::STRICT_UINT_TO_FP.
2363
2364unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2365 if ((Subtarget->hasVLX())) {
2366 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2367 }
2368 return 0;
2369}
2370
2371unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2372 if ((Subtarget->hasVLX())) {
2373 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2374 }
2375 return 0;
2376}
2377
2378unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2379switch (RetVT.SimpleTy) {
2380 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
2381 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
2382 default: return 0;
2383}
2384}
2385
2386unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2387 if ((Subtarget->hasVLX())) {
2388 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2389 }
2390 return 0;
2391}
2392
2393unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2394 if ((Subtarget->hasAVX512())) {
2395 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2396 }
2397 return 0;
2398}
2399
2400unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2401switch (RetVT.SimpleTy) {
2402 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
2403 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
2404 default: return 0;
2405}
2406}
2407
2408unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2409 if (RetVT.SimpleTy != MVT::v16f32)
2410 return 0;
2411 if ((Subtarget->hasAVX512())) {
2412 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2413 }
2414 return 0;
2415}
2416
2417unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2418 if (RetVT.SimpleTy != MVT::v2f64)
2419 return 0;
2420 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2421 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2422 }
2423 return 0;
2424}
2425
2426unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2427 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2428 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2429 }
2430 return 0;
2431}
2432
2433unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2434 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2435 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2436 }
2437 return 0;
2438}
2439
2440unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2441switch (RetVT.SimpleTy) {
2442 case MVT::v4f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
2443 case MVT::v4f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
2444 default: return 0;
2445}
2446}
2447
2448unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2449 if ((Subtarget->hasDQI())) {
2450 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2451 }
2452 return 0;
2453}
2454
2455unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2456 if ((Subtarget->hasDQI())) {
2457 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2458 }
2459 return 0;
2460}
2461
2462unsigned fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2463switch (RetVT.SimpleTy) {
2464 case MVT::v8f32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
2465 case MVT::v8f64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
2466 default: return 0;
2467}
2468}
2469
2470unsigned fastEmit_ISD_STRICT_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2471 switch (VT.SimpleTy) {
2472 case MVT::v4i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2473 case MVT::v8i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2474 case MVT::v16i32: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2475 case MVT::v2i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2476 case MVT::v4i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2477 case MVT::v8i64: return fastEmit_ISD_STRICT_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2478 default: return 0;
2479 }
2480}
2481
2482// FastEmit functions for ISD::TRUNCATE.
2483
2484unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2485 if (RetVT.SimpleTy != MVT::i8)
2486 return 0;
2487 if ((Subtarget->is64Bit())) {
2488 return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
2489 }
2490 return 0;
2491}
2492
2493unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
2494 if ((Subtarget->is64Bit())) {
2495 return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
2496 }
2497 return 0;
2498}
2499
2500unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
2501 return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
2502}
2503
2504unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2505switch (RetVT.SimpleTy) {
2506 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
2507 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
2508 default: return 0;
2509}
2510}
2511
2512unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
2513 return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
2514}
2515
2516unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
2517 return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
2518}
2519
2520unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
2521 return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
2522}
2523
2524unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2525switch (RetVT.SimpleTy) {
2526 case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
2527 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
2528 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
2529 default: return 0;
2530}
2531}
2532
2533unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2534 if (RetVT.SimpleTy != MVT::v16i8)
2535 return 0;
2536 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2537 return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2538 }
2539 return 0;
2540}
2541
2542unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2543 if (RetVT.SimpleTy != MVT::v32i8)
2544 return 0;
2545 if ((Subtarget->hasBWI())) {
2546 return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2547 }
2548 return 0;
2549}
2550
2551unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2552 if (RetVT.SimpleTy != MVT::v8i16)
2553 return 0;
2554 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2555 return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2556 }
2557 return 0;
2558}
2559
2560unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
2561 if ((Subtarget->hasAVX512())) {
2562 return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2563 }
2564 return 0;
2565}
2566
2567unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
2568 if ((Subtarget->hasAVX512())) {
2569 return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2570 }
2571 return 0;
2572}
2573
2574unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2575switch (RetVT.SimpleTy) {
2576 case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
2577 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
2578 default: return 0;
2579}
2580}
2581
2582unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2583 if (RetVT.SimpleTy != MVT::v4i32)
2584 return 0;
2585 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2586 return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2587 }
2588 return 0;
2589}
2590
2591unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2592 if ((Subtarget->hasAVX512())) {
2593 return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
2594 }
2595 return 0;
2596}
2597
2598unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2599 if ((Subtarget->hasAVX512())) {
2600 return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2601 }
2602 return 0;
2603}
2604
2605unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2606switch (RetVT.SimpleTy) {
2607 case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
2608 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
2609 default: return 0;
2610}
2611}
2612
2613unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2614 switch (VT.SimpleTy) {
2615 case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
2616 case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
2617 case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
2618 case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2619 case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
2620 case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2621 case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2622 case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2623 case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2624 default: return 0;
2625 }
2626}
2627
2628// FastEmit functions for ISD::UINT_TO_FP.
2629
2630unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2631 if ((Subtarget->hasVLX())) {
2632 return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2633 }
2634 return 0;
2635}
2636
2637unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2638 if ((Subtarget->hasVLX())) {
2639 return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2640 }
2641 return 0;
2642}
2643
2644unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2645switch (RetVT.SimpleTy) {
2646 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
2647 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
2648 default: return 0;
2649}
2650}
2651
2652unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2653 if ((Subtarget->hasVLX())) {
2654 return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2655 }
2656 return 0;
2657}
2658
2659unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2660 if ((Subtarget->hasAVX512())) {
2661 return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2662 }
2663 return 0;
2664}
2665
2666unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2667switch (RetVT.SimpleTy) {
2668 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
2669 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
2670 default: return 0;
2671}
2672}
2673
2674unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2675 if (RetVT.SimpleTy != MVT::v16f32)
2676 return 0;
2677 if ((Subtarget->hasAVX512())) {
2678 return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2679 }
2680 return 0;
2681}
2682
2683unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2684 if (RetVT.SimpleTy != MVT::v2f64)
2685 return 0;
2686 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2687 return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2688 }
2689 return 0;
2690}
2691
2692unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
2693 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2694 return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2695 }
2696 return 0;
2697}
2698
2699unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
2700 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
2701 return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2702 }
2703 return 0;
2704}
2705
2706unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2707switch (RetVT.SimpleTy) {
2708 case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
2709 case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
2710 default: return 0;
2711}
2712}
2713
2714unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
2715 if ((Subtarget->hasDQI())) {
2716 return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
2717 }
2718 return 0;
2719}
2720
2721unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
2722 if ((Subtarget->hasDQI())) {
2723 return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2724 }
2725 return 0;
2726}
2727
2728unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2729switch (RetVT.SimpleTy) {
2730 case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
2731 case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
2732 default: return 0;
2733}
2734}
2735
2736unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2737 switch (VT.SimpleTy) {
2738 case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2739 case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2740 case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
2741 case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
2742 case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
2743 case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
2744 default: return 0;
2745 }
2746}
2747
2748// FastEmit functions for ISD::ZERO_EXTEND.
2749
2750unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2751 if (RetVT.SimpleTy != MVT::i32)
2752 return 0;
2753 return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
2754}
2755
2756unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2757 if (RetVT.SimpleTy != MVT::i32)
2758 return 0;
2759 return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
2760}
2761
2762unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
2763 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
2764 return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2765 }
2766 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2767 return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2768 }
2769 return 0;
2770}
2771
2772unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
2773 if ((Subtarget->hasAVX512())) {
2774 return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2775 }
2776 return 0;
2777}
2778
2779unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2780switch (RetVT.SimpleTy) {
2781 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
2782 case MVT::v16i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
2783 default: return 0;
2784}
2785}
2786
2787unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2788 if (RetVT.SimpleTy != MVT::v32i16)
2789 return 0;
2790 if ((Subtarget->hasBWI())) {
2791 return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2792 }
2793 return 0;
2794}
2795
2796unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2797 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2798 return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2799 }
2800 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2801 return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2802 }
2803 return 0;
2804}
2805
2806unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2807 if ((Subtarget->hasAVX512())) {
2808 return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2809 }
2810 return 0;
2811}
2812
2813unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2814switch (RetVT.SimpleTy) {
2815 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
2816 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
2817 default: return 0;
2818}
2819}
2820
2821unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2822 if (RetVT.SimpleTy != MVT::v16i32)
2823 return 0;
2824 if ((Subtarget->hasAVX512())) {
2825 return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2826 }
2827 return 0;
2828}
2829
2830unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2831 if (RetVT.SimpleTy != MVT::v4i64)
2832 return 0;
2833 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2834 return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2835 }
2836 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2837 return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2838 }
2839 return 0;
2840}
2841
2842unsigned fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2843 if (RetVT.SimpleTy != MVT::v8i64)
2844 return 0;
2845 if ((Subtarget->hasAVX512())) {
2846 return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2847 }
2848 return 0;
2849}
2850
2851unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
2852 switch (VT.SimpleTy) {
2853 case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
2854 case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
2855 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
2856 case MVT::v32i8: return fastEmit_ISD_ZERO_EXTEND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
2857 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
2858 case MVT::v16i16: return fastEmit_ISD_ZERO_EXTEND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
2859 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
2860 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
2861 default: return 0;
2862 }
2863}
2864
2865// FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
2866
2867unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
2868 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2869 return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2870 }
2871 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
2872 return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
2873 }
2874 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
2875 return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2876 }
2877 return 0;
2878}
2879
2880unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2881 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2882 return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2883 }
2884 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2885 return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2886 }
2887 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2888 return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2889 }
2890 return 0;
2891}
2892
2893unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
2894 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2895 return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2896 }
2897 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2898 return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2899 }
2900 return 0;
2901}
2902
2903unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2904 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2905 return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2906 }
2907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2908 return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2909 }
2910 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2911 return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2912 }
2913 return 0;
2914}
2915
2916unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2917 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2918 return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2919 }
2920 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2921 return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2922 }
2923 return 0;
2924}
2925
2926unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
2927 if ((Subtarget->hasAVX512())) {
2928 return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
2929 }
2930 return 0;
2931}
2932
2933unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2934switch (RetVT.SimpleTy) {
2935 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
2936 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
2937 case MVT::v8i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
2938 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
2939 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
2940 case MVT::v8i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
2941 default: return 0;
2942}
2943}
2944
2945unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
2946 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2947 return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2948 }
2949 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2950 return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
2951 }
2952 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2953 return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2954 }
2955 return 0;
2956}
2957
2958unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
2959 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2960 return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2961 }
2962 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2963 return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2964 }
2965 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2966 return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
2967 }
2968 return 0;
2969}
2970
2971unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
2972 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
2973 return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
2974 }
2975 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
2976 return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
2977 }
2978 return 0;
2979}
2980
2981unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2982switch (RetVT.SimpleTy) {
2983 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
2984 case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
2985 case MVT::v4i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
2986 default: return 0;
2987}
2988}
2989
2990unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
2991 if (RetVT.SimpleTy != MVT::v2i64)
2992 return 0;
2993 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
2994 return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2995 }
2996 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
2997 return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
2998 }
2999 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
3000 return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3001 }
3002 return 0;
3003}
3004
3005unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3006 switch (VT.SimpleTy) {
3007 case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
3008 case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
3009 case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3010 default: return 0;
3011 }
3012}
3013
3014// FastEmit functions for X86ISD::CALL.
3015
3016unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3017 if (RetVT.SimpleTy != MVT::isVoid)
3018 return 0;
3019 if ((!Subtarget->is64Bit())) {
3020 return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
3021 }
3022 return 0;
3023}
3024
3025unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3026 if (RetVT.SimpleTy != MVT::isVoid)
3027 return 0;
3028 if ((!Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3029 return fastEmitInst_r(X86::INDIRECT_THUNK_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
3030 }
3031 if ((!Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3032 return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
3033 }
3034 return 0;
3035}
3036
3037unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3038 if (RetVT.SimpleTy != MVT::isVoid)
3039 return 0;
3040 if ((Subtarget->is64Bit()) && (Subtarget->useIndirectThunkCalls())) {
3041 return fastEmitInst_r(X86::INDIRECT_THUNK_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
3042 }
3043 if ((Subtarget->is64Bit()) && (!Subtarget->useIndirectThunkCalls())) {
3044 return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
3045 }
3046 return 0;
3047}
3048
3049unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3050 switch (VT.SimpleTy) {
3051 case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
3052 case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
3053 case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
3054 default: return 0;
3055 }
3056}
3057
3058// FastEmit functions for X86ISD::CONFLICT.
3059
3060unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3061 if (RetVT.SimpleTy != MVT::v4i32)
3062 return 0;
3063 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3064 return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3065 }
3066 return 0;
3067}
3068
3069unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3070 if (RetVT.SimpleTy != MVT::v8i32)
3071 return 0;
3072 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3073 return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3074 }
3075 return 0;
3076}
3077
3078unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3079 if (RetVT.SimpleTy != MVT::v16i32)
3080 return 0;
3081 if ((Subtarget->hasCDI())) {
3082 return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3083 }
3084 return 0;
3085}
3086
3087unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3088 if (RetVT.SimpleTy != MVT::v2i64)
3089 return 0;
3090 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3091 return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3092 }
3093 return 0;
3094}
3095
3096unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3097 if (RetVT.SimpleTy != MVT::v4i64)
3098 return 0;
3099 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
3100 return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3101 }
3102 return 0;
3103}
3104
3105unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3106 if (RetVT.SimpleTy != MVT::v8i64)
3107 return 0;
3108 if ((Subtarget->hasCDI())) {
3109 return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3110 }
3111 return 0;
3112}
3113
3114unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3115 switch (VT.SimpleTy) {
3116 case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3117 case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
3118 case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
3119 case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3120 case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
3121 case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
3122 default: return 0;
3123 }
3124}
3125
3126// FastEmit functions for X86ISD::CVTNEPS2BF16.
3127
3128unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3129 if (RetVT.SimpleTy != MVT::v8i16)
3130 return 0;
3131 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3132 return fastEmitInst_r(X86::VCVTNEPS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3133 }
3134 return 0;
3135}
3136
3137unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3138 if (RetVT.SimpleTy != MVT::v8i16)
3139 return 0;
3140 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
3141 return fastEmitInst_r(X86::VCVTNEPS2BF16Z256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3142 }
3143 return 0;
3144}
3145
3146unsigned fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3147 if (RetVT.SimpleTy != MVT::v16i16)
3148 return 0;
3149 if ((Subtarget->hasBF16())) {
3150 return fastEmitInst_r(X86::VCVTNEPS2BF16Zrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3151 }
3152 return 0;
3153}
3154
3155unsigned fastEmit_X86ISD_CVTNEPS2BF16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3156 switch (VT.SimpleTy) {
3157 case MVT::v4f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3158 case MVT::v8f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3159 case MVT::v16f32: return fastEmit_X86ISD_CVTNEPS2BF16_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3160 default: return 0;
3161 }
3162}
3163
3164// FastEmit functions for X86ISD::CVTP2SI.
3165
3166unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3167 if ((Subtarget->hasVLX())) {
3168 return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3169 }
3170 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3171 return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3172 }
3173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3174 return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3175 }
3176 return 0;
3177}
3178
3179unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3180 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3181 return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3182 }
3183 return 0;
3184}
3185
3186unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3187 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3188 return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3189 }
3190 return 0;
3191}
3192
3193unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3194switch (RetVT.SimpleTy) {
3195 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3196 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3197 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3198 default: return 0;
3199}
3200}
3201
3202unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3203 if ((Subtarget->hasVLX())) {
3204 return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3205 }
3206 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3207 return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3208 }
3209 return 0;
3210}
3211
3212unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3213 if ((Subtarget->hasDQI())) {
3214 return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3215 }
3216 return 0;
3217}
3218
3219unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3220switch (RetVT.SimpleTy) {
3221 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3222 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3223 default: return 0;
3224}
3225}
3226
3227unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3228 if (RetVT.SimpleTy != MVT::v16i32)
3229 return 0;
3230 if ((Subtarget->hasAVX512())) {
3231 return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3232 }
3233 return 0;
3234}
3235
3236unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3237 if ((Subtarget->hasVLX())) {
3238 return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3239 }
3240 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3241 return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3242 }
3243 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3244 return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3245 }
3246 return 0;
3247}
3248
3249unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3250 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3251 return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3252 }
3253 return 0;
3254}
3255
3256unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3257switch (RetVT.SimpleTy) {
3258 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3259 case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3260 default: return 0;
3261}
3262}
3263
3264unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3265 if ((Subtarget->hasVLX())) {
3266 return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3267 }
3268 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3269 return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
3270 }
3271 return 0;
3272}
3273
3274unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3275 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3276 return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3277 }
3278 return 0;
3279}
3280
3281unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3282switch (RetVT.SimpleTy) {
3283 case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3284 case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3285 default: return 0;
3286}
3287}
3288
3289unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3290 if ((Subtarget->hasAVX512())) {
3291 return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3292 }
3293 return 0;
3294}
3295
3296unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3297 if ((Subtarget->hasDQI())) {
3298 return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3299 }
3300 return 0;
3301}
3302
3303unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3304switch (RetVT.SimpleTy) {
3305 case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3306 case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3307 default: return 0;
3308}
3309}
3310
3311unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3312 switch (VT.SimpleTy) {
3313 case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3314 case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3315 case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3316 case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3317 case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3318 case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3319 default: return 0;
3320 }
3321}
3322
3323// FastEmit functions for X86ISD::CVTP2UI.
3324
3325unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3326 if ((Subtarget->hasVLX())) {
3327 return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3328 }
3329 return 0;
3330}
3331
3332unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3333 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3334 return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3335 }
3336 return 0;
3337}
3338
3339unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3340 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3341 return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3342 }
3343 return 0;
3344}
3345
3346unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3347switch (RetVT.SimpleTy) {
3348 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3349 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3350 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3351 default: return 0;
3352}
3353}
3354
3355unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3356 if ((Subtarget->hasVLX())) {
3357 return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3358 }
3359 return 0;
3360}
3361
3362unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3363 if ((Subtarget->hasDQI())) {
3364 return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3365 }
3366 return 0;
3367}
3368
3369unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3370switch (RetVT.SimpleTy) {
3371 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3372 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3373 default: return 0;
3374}
3375}
3376
3377unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3378 if (RetVT.SimpleTy != MVT::v16i32)
3379 return 0;
3380 if ((Subtarget->hasAVX512())) {
3381 return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3382 }
3383 return 0;
3384}
3385
3386unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3387 if ((Subtarget->hasVLX())) {
3388 return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3389 }
3390 return 0;
3391}
3392
3393unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3394 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3395 return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3396 }
3397 return 0;
3398}
3399
3400unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3401switch (RetVT.SimpleTy) {
3402 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3403 case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3404 default: return 0;
3405}
3406}
3407
3408unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3409 if ((Subtarget->hasVLX())) {
3410 return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3411 }
3412 return 0;
3413}
3414
3415unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3416 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3417 return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3418 }
3419 return 0;
3420}
3421
3422unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3423switch (RetVT.SimpleTy) {
3424 case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3425 case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3426 default: return 0;
3427}
3428}
3429
3430unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3431 if ((Subtarget->hasAVX512())) {
3432 return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3433 }
3434 return 0;
3435}
3436
3437unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3438 if ((Subtarget->hasDQI())) {
3439 return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3440 }
3441 return 0;
3442}
3443
3444unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3445switch (RetVT.SimpleTy) {
3446 case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3447 case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3448 default: return 0;
3449}
3450}
3451
3452unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3453 switch (VT.SimpleTy) {
3454 case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3455 case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3456 case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3457 case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3458 case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3459 case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3460 default: return 0;
3461 }
3462}
3463
3464// FastEmit functions for X86ISD::CVTPH2PS.
3465
3466unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
3467 if ((Subtarget->hasVLX())) {
3468 return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3469 }
3470 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
3471 return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
3472 }
3473 return 0;
3474}
3475
3476unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
3477 if ((Subtarget->hasVLX())) {
3478 return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3479 }
3480 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
3481 return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3482 }
3483 return 0;
3484}
3485
3486unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3487switch (RetVT.SimpleTy) {
3488 case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
3489 case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
3490 default: return 0;
3491}
3492}
3493
3494unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3495 if (RetVT.SimpleTy != MVT::v16f32)
3496 return 0;
3497 if ((Subtarget->hasAVX512())) {
3498 return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3499 }
3500 return 0;
3501}
3502
3503unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3504 switch (VT.SimpleTy) {
3505 case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
3506 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
3507 default: return 0;
3508 }
3509}
3510
3511// FastEmit functions for X86ISD::CVTPH2PS_SAE.
3512
3513unsigned fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3514 if (RetVT.SimpleTy != MVT::v16f32)
3515 return 0;
3516 if ((Subtarget->hasAVX512())) {
3517 return fastEmitInst_r(X86::VCVTPH2PSZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
3518 }
3519 return 0;
3520}
3521
3522unsigned fastEmit_X86ISD_CVTPH2PS_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3523 switch (VT.SimpleTy) {
3524 case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_SAE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
3525 default: return 0;
3526 }
3527}
3528
3529// FastEmit functions for X86ISD::CVTS2SI.
3530
3531unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3532 if ((Subtarget->hasAVX512())) {
3533 return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3534 }
3535 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3536 return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3537 }
3538 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3539 return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3540 }
3541 return 0;
3542}
3543
3544unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3545 if ((Subtarget->hasAVX512())) {
3546 return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3547 }
3548 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
3549 return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3550 }
3551 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3552 return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3553 }
3554 return 0;
3555}
3556
3557unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3558switch (RetVT.SimpleTy) {
3559 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3560 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3561 default: return 0;
3562}
3563}
3564
3565unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3566 if ((Subtarget->hasAVX512())) {
3567 return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3568 }
3569 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3570 return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3571 }
3572 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3573 return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3574 }
3575 return 0;
3576}
3577
3578unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3579 if ((Subtarget->hasAVX512())) {
3580 return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3581 }
3582 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3583 return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3584 }
3585 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
3586 return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3587 }
3588 return 0;
3589}
3590
3591unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3592switch (RetVT.SimpleTy) {
3593 case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3594 case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3595 default: return 0;
3596}
3597}
3598
3599unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3600 switch (VT.SimpleTy) {
3601 case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3602 case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3603 default: return 0;
3604 }
3605}
3606
3607// FastEmit functions for X86ISD::CVTS2UI.
3608
3609unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3610 if ((Subtarget->hasAVX512())) {
3611 return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3612 }
3613 return 0;
3614}
3615
3616unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3617 if ((Subtarget->hasAVX512())) {
3618 return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3619 }
3620 return 0;
3621}
3622
3623unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3624switch (RetVT.SimpleTy) {
3625 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
3626 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
3627 default: return 0;
3628}
3629}
3630
3631unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
3632 if ((Subtarget->hasAVX512())) {
3633 return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
3634 }
3635 return 0;
3636}
3637
3638unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
3639 if ((Subtarget->hasAVX512())) {
3640 return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
3641 }
3642 return 0;
3643}
3644
3645unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3646switch (RetVT.SimpleTy) {
3647 case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
3648 case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
3649 default: return 0;
3650}
3651}
3652
3653unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3654 switch (VT.SimpleTy) {
3655 case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3656 case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3657 default: return 0;
3658 }
3659}
3660
3661// FastEmit functions for X86ISD::CVTSI2P.
3662
3663unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3664 if (RetVT.SimpleTy != MVT::v2f64)
3665 return 0;
3666 if ((Subtarget->hasVLX())) {
3667 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3668 }
3669 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3670 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
3671 }
3672 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3673 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
3674 }
3675 return 0;
3676}
3677
3678unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3679 if (RetVT.SimpleTy != MVT::v4f32)
3680 return 0;
3681 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3682 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3683 }
3684 return 0;
3685}
3686
3687unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3688 switch (VT.SimpleTy) {
3689 case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
3690 case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
3691 default: return 0;
3692 }
3693}
3694
3695// FastEmit functions for X86ISD::CVTTP2SI.
3696
3697unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3698 if ((Subtarget->hasVLX())) {
3699 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3700 }
3701 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3702 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3703 }
3704 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3705 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3706 }
3707 return 0;
3708}
3709
3710unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3711 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3712 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3713 }
3714 return 0;
3715}
3716
3717unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3718 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3719 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3720 }
3721 return 0;
3722}
3723
3724unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3725switch (RetVT.SimpleTy) {
3726 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3727 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3728 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3729 default: return 0;
3730}
3731}
3732
3733unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3734 if ((Subtarget->hasVLX())) {
3735 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3736 }
3737 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3738 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
3739 }
3740 return 0;
3741}
3742
3743unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3744 if ((Subtarget->hasDQI())) {
3745 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3746 }
3747 return 0;
3748}
3749
3750unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3751switch (RetVT.SimpleTy) {
3752 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3753 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3754 default: return 0;
3755}
3756}
3757
3758unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3759 if (RetVT.SimpleTy != MVT::v16i32)
3760 return 0;
3761 if ((Subtarget->hasAVX512())) {
3762 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3763 }
3764 return 0;
3765}
3766
3767unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3768 if ((Subtarget->hasVLX())) {
3769 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3770 }
3771 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
3772 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3773 }
3774 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3775 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
3776 }
3777 return 0;
3778}
3779
3780unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3781 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3782 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3783 }
3784 return 0;
3785}
3786
3787unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3788switch (RetVT.SimpleTy) {
3789 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3790 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3791 default: return 0;
3792}
3793}
3794
3795unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3796 if ((Subtarget->hasVLX())) {
3797 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3798 }
3799 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
3800 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
3801 }
3802 return 0;
3803}
3804
3805unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3806 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3807 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3808 }
3809 return 0;
3810}
3811
3812unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3813switch (RetVT.SimpleTy) {
3814 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
3815 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
3816 default: return 0;
3817}
3818}
3819
3820unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3821 if ((Subtarget->hasAVX512())) {
3822 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
3823 }
3824 return 0;
3825}
3826
3827unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3828 if ((Subtarget->hasDQI())) {
3829 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3830 }
3831 return 0;
3832}
3833
3834unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3835switch (RetVT.SimpleTy) {
3836 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3837 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3838 default: return 0;
3839}
3840}
3841
3842unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3843 switch (VT.SimpleTy) {
3844 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
3845 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3846 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3847 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
3848 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
3849 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3850 default: return 0;
3851 }
3852}
3853
3854// FastEmit functions for X86ISD::CVTTP2SI_SAE.
3855
3856unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3857 if (RetVT.SimpleTy != MVT::v8i64)
3858 return 0;
3859 if ((Subtarget->hasDQI())) {
3860 return fastEmitInst_r(X86::VCVTTPS2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
3861 }
3862 return 0;
3863}
3864
3865unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3866 if (RetVT.SimpleTy != MVT::v16i32)
3867 return 0;
3868 if ((Subtarget->hasAVX512())) {
3869 return fastEmitInst_r(X86::VCVTTPS2DQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
3870 }
3871 return 0;
3872}
3873
3874unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3875 if ((Subtarget->hasAVX512())) {
3876 return fastEmitInst_r(X86::VCVTTPD2DQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill);
3877 }
3878 return 0;
3879}
3880
3881unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3882 if ((Subtarget->hasDQI())) {
3883 return fastEmitInst_r(X86::VCVTTPD2QQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
3884 }
3885 return 0;
3886}
3887
3888unsigned fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3889switch (RetVT.SimpleTy) {
3890 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
3891 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
3892 default: return 0;
3893}
3894}
3895
3896unsigned fastEmit_X86ISD_CVTTP2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
3897 switch (VT.SimpleTy) {
3898 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
3899 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
3900 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
3901 default: return 0;
3902 }
3903}
3904
3905// FastEmit functions for X86ISD::CVTTP2UI.
3906
3907unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3908 if ((Subtarget->hasVLX())) {
3909 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3910 }
3911 return 0;
3912}
3913
3914unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3915 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3916 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3917 }
3918 return 0;
3919}
3920
3921unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3922 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3923 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3924 }
3925 return 0;
3926}
3927
3928unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3929switch (RetVT.SimpleTy) {
3930 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
3931 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
3932 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
3933 default: return 0;
3934}
3935}
3936
3937unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
3938 if ((Subtarget->hasVLX())) {
3939 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
3940 }
3941 return 0;
3942}
3943
3944unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
3945 if ((Subtarget->hasDQI())) {
3946 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3947 }
3948 return 0;
3949}
3950
3951unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3952switch (RetVT.SimpleTy) {
3953 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
3954 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
3955 default: return 0;
3956}
3957}
3958
3959unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3960 if (RetVT.SimpleTy != MVT::v16i32)
3961 return 0;
3962 if ((Subtarget->hasAVX512())) {
3963 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
3964 }
3965 return 0;
3966}
3967
3968unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3969 if ((Subtarget->hasVLX())) {
3970 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3971 }
3972 return 0;
3973}
3974
3975unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
3976 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3977 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3978 }
3979 return 0;
3980}
3981
3982unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
3983switch (RetVT.SimpleTy) {
3984 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
3985 case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
3986 default: return 0;
3987}
3988}
3989
3990unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
3991 if ((Subtarget->hasVLX())) {
3992 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
3993 }
3994 return 0;
3995}
3996
3997unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
3998 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
3999 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4000 }
4001 return 0;
4002}
4003
4004unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4005switch (RetVT.SimpleTy) {
4006 case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
4007 case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
4008 default: return 0;
4009}
4010}
4011
4012unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4013 if ((Subtarget->hasAVX512())) {
4014 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
4015 }
4016 return 0;
4017}
4018
4019unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4020 if ((Subtarget->hasDQI())) {
4021 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4022 }
4023 return 0;
4024}
4025
4026unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4027switch (RetVT.SimpleTy) {
4028 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
4029 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
4030 default: return 0;
4031}
4032}
4033
4034unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4035 switch (VT.SimpleTy) {
4036 case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4037 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4038 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4039 case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4040 case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4041 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4042 default: return 0;
4043 }
4044}
4045
4046// FastEmit functions for X86ISD::CVTTP2UI_SAE.
4047
4048unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4049 if (RetVT.SimpleTy != MVT::v8i64)
4050 return 0;
4051 if ((Subtarget->hasDQI())) {
4052 return fastEmitInst_r(X86::VCVTTPS2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
4053 }
4054 return 0;
4055}
4056
4057unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4058 if (RetVT.SimpleTy != MVT::v16i32)
4059 return 0;
4060 if ((Subtarget->hasAVX512())) {
4061 return fastEmitInst_r(X86::VCVTTPS2UDQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
4062 }
4063 return 0;
4064}
4065
4066unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
4067 if ((Subtarget->hasAVX512())) {
4068 return fastEmitInst_r(X86::VCVTTPD2UDQZrrb, &X86::VR256XRegClass, Op0, Op0IsKill);
4069 }
4070 return 0;
4071}
4072
4073unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
4074 if ((Subtarget->hasDQI())) {
4075 return fastEmitInst_r(X86::VCVTTPD2UQQZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
4076 }
4077 return 0;
4078}
4079
4080unsigned fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4081switch (RetVT.SimpleTy) {
4082 case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
4083 case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
4084 default: return 0;
4085}
4086}
4087
4088unsigned fastEmit_X86ISD_CVTTP2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4089 switch (VT.SimpleTy) {
4090 case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4091 case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4092 case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4093 default: return 0;
4094 }
4095}
4096
4097// FastEmit functions for X86ISD::CVTTS2SI.
4098
4099unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4100 if ((Subtarget->hasAVX512())) {
4101 return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4102 }
4103 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4104 return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4105 }
4106 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4107 return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4108 }
4109 return 0;
4110}
4111
4112unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4113 if ((Subtarget->hasAVX512())) {
4114 return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4115 }
4116 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4117 return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4118 }
4119 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4120 return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4121 }
4122 return 0;
4123}
4124
4125unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4126switch (RetVT.SimpleTy) {
4127 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
4128 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
4129 default: return 0;
4130}
4131}
4132
4133unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4134 if ((Subtarget->hasAVX512())) {
4135 return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4136 }
4137 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4138 return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4139 }
4140 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4141 return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4142 }
4143 return 0;
4144}
4145
4146unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4147 if ((Subtarget->hasAVX512())) {
4148 return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4149 }
4150 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4151 return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4152 }
4153 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
4154 return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4155 }
4156 return 0;
4157}
4158
4159unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4160switch (RetVT.SimpleTy) {
4161 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
4162 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
4163 default: return 0;
4164}
4165}
4166
4167unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4168 switch (VT.SimpleTy) {
4169 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4170 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4171 default: return 0;
4172 }
4173}
4174
4175// FastEmit functions for X86ISD::CVTTS2SI_SAE.
4176
4177unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4178 if ((Subtarget->hasAVX512())) {
4179 return fastEmitInst_r(X86::VCVTTSS2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4180 }
4181 return 0;
4182}
4183
4184unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4185 if ((Subtarget->hasAVX512())) {
4186 return fastEmitInst_r(X86::VCVTTSS2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4187 }
4188 return 0;
4189}
4190
4191unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4192switch (RetVT.SimpleTy) {
4193 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
4194 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
4195 default: return 0;
4196}
4197}
4198
4199unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4200 if ((Subtarget->hasAVX512())) {
4201 return fastEmitInst_r(X86::VCVTTSD2SIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4202 }
4203 return 0;
4204}
4205
4206unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4207 if ((Subtarget->hasAVX512())) {
4208 return fastEmitInst_r(X86::VCVTTSD2SI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4209 }
4210 return 0;
4211}
4212
4213unsigned fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4214switch (RetVT.SimpleTy) {
4215 case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
4216 case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
4217 default: return 0;
4218}
4219}
4220
4221unsigned fastEmit_X86ISD_CVTTS2SI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4222 switch (VT.SimpleTy) {
4223 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4224 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_SAE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4225 default: return 0;
4226 }
4227}
4228
4229// FastEmit functions for X86ISD::CVTTS2UI.
4230
4231unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4232 if ((Subtarget->hasAVX512())) {
4233 return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4234 }
4235 return 0;
4236}
4237
4238unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4239 if ((Subtarget->hasAVX512())) {
4240 return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4241 }
4242 return 0;
4243}
4244
4245unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4246switch (RetVT.SimpleTy) {
4247 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
4248 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
4249 default: return 0;
4250}
4251}
4252
4253unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4254 if ((Subtarget->hasAVX512())) {
4255 return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4256 }
4257 return 0;
4258}
4259
4260unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4261 if ((Subtarget->hasAVX512())) {
4262 return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4263 }
4264 return 0;
4265}
4266
4267unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4268switch (RetVT.SimpleTy) {
4269 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
4270 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
4271 default: return 0;
4272}
4273}
4274
4275unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4276 switch (VT.SimpleTy) {
4277 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4278 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4279 default: return 0;
4280 }
4281}
4282
4283// FastEmit functions for X86ISD::CVTTS2UI_SAE.
4284
4285unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4286 if ((Subtarget->hasAVX512())) {
4287 return fastEmitInst_r(X86::VCVTTSS2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4288 }
4289 return 0;
4290}
4291
4292unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4293 if ((Subtarget->hasAVX512())) {
4294 return fastEmitInst_r(X86::VCVTTSS2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4295 }
4296 return 0;
4297}
4298
4299unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4300switch (RetVT.SimpleTy) {
4301 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
4302 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
4303 default: return 0;
4304}
4305}
4306
4307unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
4308 if ((Subtarget->hasAVX512())) {
4309 return fastEmitInst_r(X86::VCVTTSD2USIZrrb_Int, &X86::GR32RegClass, Op0, Op0IsKill);
4310 }
4311 return 0;
4312}
4313
4314unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
4315 if ((Subtarget->hasAVX512())) {
4316 return fastEmitInst_r(X86::VCVTTSD2USI64Zrrb_Int, &X86::GR64RegClass, Op0, Op0IsKill);
4317 }
4318 return 0;
4319}
4320
4321unsigned fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4322switch (RetVT.SimpleTy) {
4323 case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
4324 case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
4325 default: return 0;
4326}
4327}
4328
4329unsigned fastEmit_X86ISD_CVTTS2UI_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4330 switch (VT.SimpleTy) {
4331 case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4332 case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_SAE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4333 default: return 0;
4334 }
4335}
4336
4337// FastEmit functions for X86ISD::CVTUI2P.
4338
4339unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4340 if (RetVT.SimpleTy != MVT::v2f64)
4341 return 0;
4342 if ((Subtarget->hasVLX())) {
4343 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4344 }
4345 return 0;
4346}
4347
4348unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4349 if (RetVT.SimpleTy != MVT::v4f32)
4350 return 0;
4351 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
4352 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4353 }
4354 return 0;
4355}
4356
4357unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4358 switch (VT.SimpleTy) {
4359 case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4360 case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4361 default: return 0;
4362 }
4363}
4364
4365// FastEmit functions for X86ISD::EH_RETURN.
4366
4367unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4368 if (RetVT.SimpleTy != MVT::isVoid)
4369 return 0;
4370 return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
4371}
4372
4373unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4374 if (RetVT.SimpleTy != MVT::isVoid)
4375 return 0;
4376 return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
4377}
4378
4379unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4380 switch (VT.SimpleTy) {
4381 case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
4382 case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
4383 default: return 0;
4384 }
4385}
4386
4387// FastEmit functions for X86ISD::EXP2.
4388
4389unsigned fastEmit_X86ISD_EXP2_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4390 if (RetVT.SimpleTy != MVT::v16f32)
4391 return 0;
4392 if ((Subtarget->hasERI())) {
4393 return fastEmitInst_r(X86::VEXP2PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4394 }
4395 return 0;
4396}
4397
4398unsigned fastEmit_X86ISD_EXP2_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4399 if (RetVT.SimpleTy != MVT::v8f64)
4400 return 0;
4401 if ((Subtarget->hasERI())) {
4402 return fastEmitInst_r(X86::VEXP2PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4403 }
4404 return 0;
4405}
4406
4407unsigned fastEmit_X86ISD_EXP2_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4408 switch (VT.SimpleTy) {
4409 case MVT::v16f32: return fastEmit_X86ISD_EXP2_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4410 case MVT::v8f64: return fastEmit_X86ISD_EXP2_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4411 default: return 0;
4412 }
4413}
4414
4415// FastEmit functions for X86ISD::EXP2_SAE.
4416
4417unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4418 if (RetVT.SimpleTy != MVT::v16f32)
4419 return 0;
4420 if ((Subtarget->hasERI())) {
4421 return fastEmitInst_r(X86::VEXP2PSZrb, &X86::VR512RegClass, Op0, Op0IsKill);
4422 }
4423 return 0;
4424}
4425
4426unsigned fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4427 if (RetVT.SimpleTy != MVT::v8f64)
4428 return 0;
4429 if ((Subtarget->hasERI())) {
4430 return fastEmitInst_r(X86::VEXP2PDZrb, &X86::VR512RegClass, Op0, Op0IsKill);
4431 }
4432 return 0;
4433}
4434
4435unsigned fastEmit_X86ISD_EXP2_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4436 switch (VT.SimpleTy) {
4437 case MVT::v16f32: return fastEmit_X86ISD_EXP2_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4438 case MVT::v8f64: return fastEmit_X86ISD_EXP2_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4439 default: return 0;
4440 }
4441}
4442
4443// FastEmit functions for X86ISD::FGETEXP.
4444
4445unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4446 if (RetVT.SimpleTy != MVT::v4f32)
4447 return 0;
4448 if ((Subtarget->hasVLX())) {
4449 return fastEmitInst_r(X86::VGETEXPPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4450 }
4451 return 0;
4452}
4453
4454unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4455 if (RetVT.SimpleTy != MVT::v8f32)
4456 return 0;
4457 if ((Subtarget->hasVLX())) {
4458 return fastEmitInst_r(X86::VGETEXPPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4459 }
4460 return 0;
4461}
4462
4463unsigned fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4464 if (RetVT.SimpleTy != MVT::v16f32)
4465 return 0;
4466 if ((Subtarget->hasAVX512())) {
4467 return fastEmitInst_r(X86::VGETEXPPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
4468 }
4469 return 0;
4470}
4471
4472unsigned fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4473 if (RetVT.SimpleTy != MVT::v2f64)
4474 return 0;
4475 if ((Subtarget->hasVLX())) {
4476 return fastEmitInst_r(X86::VGETEXPPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
4477 }
4478 return 0;
4479}
4480
4481unsigned fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4482 if (RetVT.SimpleTy != MVT::v4f64)
4483 return 0;
4484 if ((Subtarget->hasVLX())) {
4485 return fastEmitInst_r(X86::VGETEXPPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
4486 }
4487 return 0;
4488}
4489
4490unsigned fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4491 if (RetVT.SimpleTy != MVT::v8f64)
4492 return 0;
4493 if ((Subtarget->hasAVX512())) {
4494 return fastEmitInst_r(X86::VGETEXPPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
4495 }
4496 return 0;
4497}
4498
4499unsigned fastEmit_X86ISD_FGETEXP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4500 switch (VT.SimpleTy) {
4501 case MVT::v4f32: return fastEmit_X86ISD_FGETEXP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4502 case MVT::v8f32: return fastEmit_X86ISD_FGETEXP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4503 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4504 case MVT::v2f64: return fastEmit_X86ISD_FGETEXP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4505 case MVT::v4f64: return fastEmit_X86ISD_FGETEXP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4506 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4507 default: return 0;
4508 }
4509}
4510
4511// FastEmit functions for X86ISD::FGETEXP_SAE.
4512
4513unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4514 if (RetVT.SimpleTy != MVT::v16f32)
4515 return 0;
4516 if ((Subtarget->hasAVX512())) {
4517 return fastEmitInst_r(X86::VGETEXPPSZrb, &X86::VR512RegClass, Op0, Op0IsKill);
4518 }
4519 return 0;
4520}
4521
4522unsigned fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4523 if (RetVT.SimpleTy != MVT::v8f64)
4524 return 0;
4525 if ((Subtarget->hasAVX512())) {
4526 return fastEmitInst_r(X86::VGETEXPPDZrb, &X86::VR512RegClass, Op0, Op0IsKill);
4527 }
4528 return 0;
4529}
4530
4531unsigned fastEmit_X86ISD_FGETEXP_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4532 switch (VT.SimpleTy) {
4533 case MVT::v16f32: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4534 case MVT::v8f64: return fastEmit_X86ISD_FGETEXP_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4535 default: return 0;
4536 }
4537}
4538
4539// FastEmit functions for X86ISD::FRCP.
4540
4541unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4542 if (RetVT.SimpleTy != MVT::f32)
4543 return 0;
4544 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4545 return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
4546 }
4547 return 0;
4548}
4549
4550unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4551 if (RetVT.SimpleTy != MVT::v4f32)
4552 return 0;
4553 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4554 return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
4555 }
4556 if ((Subtarget->hasAVX())) {
4557 return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
4558 }
4559 return 0;
4560}
4561
4562unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4563 if (RetVT.SimpleTy != MVT::v8f32)
4564 return 0;
4565 if ((Subtarget->hasAVX())) {
4566 return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
4567 }
4568 return 0;
4569}
4570
4571unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4572 switch (VT.SimpleTy) {
4573 case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
4574 case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4575 case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4576 default: return 0;
4577 }
4578}
4579
4580// FastEmit functions for X86ISD::FRSQRT.
4581
4582unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4583 if (RetVT.SimpleTy != MVT::f32)
4584 return 0;
4585 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4586 return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
4587 }
4588 return 0;
4589}
4590
4591unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4592 if (RetVT.SimpleTy != MVT::v4f32)
4593 return 0;
4594 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4595 return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
4596 }
4597 if ((Subtarget->hasAVX())) {
4598 return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
4599 }
4600 return 0;
4601}
4602
4603unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4604 if (RetVT.SimpleTy != MVT::v8f32)
4605 return 0;
4606 if ((Subtarget->hasAVX())) {
4607 return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
4608 }
4609 return 0;
4610}
4611
4612unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4613 switch (VT.SimpleTy) {
4614 case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
4615 case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4616 case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4617 default: return 0;
4618 }
4619}
4620
4621// FastEmit functions for X86ISD::MMX_MOVD2W.
4622
4623unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4624 if (RetVT.SimpleTy != MVT::i32)
4625 return 0;
4626 if ((Subtarget->hasMMX())) {
4627 return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
4628 }
4629 return 0;
4630}
4631
4632unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4633 switch (VT.SimpleTy) {
4634 case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
4635 default: return 0;
4636 }
4637}
4638
4639// FastEmit functions for X86ISD::MMX_MOVW2D.
4640
4641unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4642 if (RetVT.SimpleTy != MVT::x86mmx)
4643 return 0;
4644 if ((Subtarget->hasMMX())) {
4645 return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
4646 }
4647 return 0;
4648}
4649
4650unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4651 switch (VT.SimpleTy) {
4652 case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
4653 default: return 0;
4654 }
4655}
4656
4657// FastEmit functions for X86ISD::MOVDDUP.
4658
4659unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4660 if (RetVT.SimpleTy != MVT::v2f64)
4661 return 0;
4662 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4663 return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4664 }
4665 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4666 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4667 }
4668 return 0;
4669}
4670
4671unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4672 if (RetVT.SimpleTy != MVT::v4f64)
4673 return 0;
4674 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4675 return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4676 }
4677 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4678 return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4679 }
4680 return 0;
4681}
4682
4683unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4684 if (RetVT.SimpleTy != MVT::v8f64)
4685 return 0;
4686 if ((Subtarget->hasAVX512())) {
4687 return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4688 }
4689 return 0;
4690}
4691
4692unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4693 switch (VT.SimpleTy) {
4694 case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4695 case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4696 case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
4697 default: return 0;
4698 }
4699}
4700
4701// FastEmit functions for X86ISD::MOVDQ2Q.
4702
4703unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4704 if (RetVT.SimpleTy != MVT::x86mmx)
4705 return 0;
4706 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
4707 return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
4708 }
4709 return 0;
4710}
4711
4712unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4713 switch (VT.SimpleTy) {
4714 case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4715 default: return 0;
4716 }
4717}
4718
4719// FastEmit functions for X86ISD::MOVMSK.
4720
4721unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4722 if (RetVT.SimpleTy != MVT::i32)
4723 return 0;
4724 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4725 return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
4726 }
4727 if ((Subtarget->hasAVX())) {
4728 return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
4729 }
4730 return 0;
4731}
4732
4733unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4734 if (RetVT.SimpleTy != MVT::i32)
4735 return 0;
4736 if ((Subtarget->hasAVX2())) {
4737 return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4738 }
4739 return 0;
4740}
4741
4742unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4743 if (RetVT.SimpleTy != MVT::i32)
4744 return 0;
4745 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4746 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4747 }
4748 if ((Subtarget->hasAVX())) {
4749 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4750 }
4751 return 0;
4752}
4753
4754unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4755 if (RetVT.SimpleTy != MVT::i32)
4756 return 0;
4757 if ((Subtarget->hasAVX())) {
4758 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4759 }
4760 return 0;
4761}
4762
4763unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4764 if (RetVT.SimpleTy != MVT::i32)
4765 return 0;
4766 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4767 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4768 }
4769 if ((Subtarget->hasAVX())) {
4770 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4771 }
4772 return 0;
4773}
4774
4775unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4776 if (RetVT.SimpleTy != MVT::i32)
4777 return 0;
4778 if ((Subtarget->hasAVX())) {
4779 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4780 }
4781 return 0;
4782}
4783
4784unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4785 if (RetVT.SimpleTy != MVT::i32)
4786 return 0;
4787 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
4788 return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4789 }
4790 if ((Subtarget->hasAVX())) {
4791 return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
4792 }
4793 return 0;
4794}
4795
4796unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4797 if (RetVT.SimpleTy != MVT::i32)
4798 return 0;
4799 if ((Subtarget->hasAVX())) {
4800 return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4801 }
4802 return 0;
4803}
4804
4805unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4806 if (RetVT.SimpleTy != MVT::i32)
4807 return 0;
4808 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
4809 return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4810 }
4811 if ((Subtarget->hasAVX())) {
4812 return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
4813 }
4814 return 0;
4815}
4816
4817unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4818 if (RetVT.SimpleTy != MVT::i32)
4819 return 0;
4820 if ((Subtarget->hasAVX())) {
4821 return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
4822 }
4823 return 0;
4824}
4825
4826unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4827 switch (VT.SimpleTy) {
4828 case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
4829 case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
4830 case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4831 case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4832 case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
4833 case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
4834 case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4835 case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4836 case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
4837 case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
4838 default: return 0;
4839 }
4840}
4841
4842// FastEmit functions for X86ISD::MOVQ2DQ.
4843
4844unsigned fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4845 if (RetVT.SimpleTy != MVT::v2i64)
4846 return 0;
4847 if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
4848 return fastEmitInst_r(X86::MMX_MOVQ2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
4849 }
4850 return 0;
4851}
4852
4853unsigned fastEmit_X86ISD_MOVQ2DQ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4854 switch (VT.SimpleTy) {
4855 case MVT::x86mmx: return fastEmit_X86ISD_MOVQ2DQ_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
4856 default: return 0;
4857 }
4858}
4859
4860// FastEmit functions for X86ISD::MOVSHDUP.
4861
4862unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4863 if (RetVT.SimpleTy != MVT::v4i32)
4864 return 0;
4865 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4866 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4867 }
4868 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4869 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4870 }
4871 return 0;
4872}
4873
4874unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4875 if (RetVT.SimpleTy != MVT::v8i32)
4876 return 0;
4877 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4878 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4879 }
4880 return 0;
4881}
4882
4883unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4884 if (RetVT.SimpleTy != MVT::v4f32)
4885 return 0;
4886 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4887 return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4888 }
4889 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4890 return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4891 }
4892 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4893 return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4894 }
4895 return 0;
4896}
4897
4898unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4899 if (RetVT.SimpleTy != MVT::v8f32)
4900 return 0;
4901 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4902 return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4903 }
4904 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4905 return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4906 }
4907 return 0;
4908}
4909
4910unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4911 if (RetVT.SimpleTy != MVT::v16f32)
4912 return 0;
4913 if ((Subtarget->hasAVX512())) {
4914 return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4915 }
4916 return 0;
4917}
4918
4919unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4920 switch (VT.SimpleTy) {
4921 case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4922 case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4923 case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4924 case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4925 case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4926 default: return 0;
4927 }
4928}
4929
4930// FastEmit functions for X86ISD::MOVSLDUP.
4931
4932unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4933 if (RetVT.SimpleTy != MVT::v4i32)
4934 return 0;
4935 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4936 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4937 }
4938 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4939 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4940 }
4941 return 0;
4942}
4943
4944unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4945 if (RetVT.SimpleTy != MVT::v8i32)
4946 return 0;
4947 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4948 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4949 }
4950 return 0;
4951}
4952
4953unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4954 if (RetVT.SimpleTy != MVT::v4f32)
4955 return 0;
4956 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4957 return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
4958 }
4959 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
4960 return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4961 }
4962 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4963 return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
4964 }
4965 return 0;
4966}
4967
4968unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4969 if (RetVT.SimpleTy != MVT::v8f32)
4970 return 0;
4971 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
4972 return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
4973 }
4974 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
4975 return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
4976 }
4977 return 0;
4978}
4979
4980unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
4981 if (RetVT.SimpleTy != MVT::v16f32)
4982 return 0;
4983 if ((Subtarget->hasAVX512())) {
4984 return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
4985 }
4986 return 0;
4987}
4988
4989unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
4990 switch (VT.SimpleTy) {
4991 case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
4992 case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
4993 case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
4994 case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
4995 case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
4996 default: return 0;
4997 }
4998}
4999
5000// FastEmit functions for X86ISD::NT_BRIND.
5001
5002unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5003 if (RetVT.SimpleTy != MVT::isVoid)
5004 return 0;
5005 if ((!Subtarget->is64Bit())) {
5006 return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
5007 }
5008 return 0;
5009}
5010
5011unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5012 if (RetVT.SimpleTy != MVT::isVoid)
5013 return 0;
5014 if ((!Subtarget->is64Bit())) {
5015 return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
5016 }
5017 return 0;
5018}
5019
5020unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5021 if (RetVT.SimpleTy != MVT::isVoid)
5022 return 0;
5023 if ((Subtarget->is64Bit())) {
5024 return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
5025 }
5026 return 0;
5027}
5028
5029unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5030 switch (VT.SimpleTy) {
5031 case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
5032 case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
5033 case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
5034 default: return 0;
5035 }
5036}
5037
5038// FastEmit functions for X86ISD::NT_CALL.
5039
5040unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5041 if (RetVT.SimpleTy != MVT::isVoid)
5042 return 0;
5043 if ((!Subtarget->is64Bit())) {
5044 return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
5045 }
5046 return 0;
5047}
5048
5049unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5050 if (RetVT.SimpleTy != MVT::isVoid)
5051 return 0;
5052 if ((!Subtarget->is64Bit())) {
5053 return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
5054 }
5055 return 0;
5056}
5057
5058unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5059 if (RetVT.SimpleTy != MVT::isVoid)
5060 return 0;
5061 if ((Subtarget->is64Bit())) {
5062 return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
5063 }
5064 return 0;
5065}
5066
5067unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5068 switch (VT.SimpleTy) {
5069 case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
5070 case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
5071 case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
5072 default: return 0;
5073 }
5074}
5075
5076// FastEmit functions for X86ISD::PHMINPOS.
5077
5078unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5079 if (RetVT.SimpleTy != MVT::v8i16)
5080 return 0;
5081 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
5082 return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
5083 }
5084 if ((Subtarget->hasAVX())) {
5085 return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
5086 }
5087 return 0;
5088}
5089
5090unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5091 switch (VT.SimpleTy) {
5092 case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5093 default: return 0;
5094 }
5095}
5096
5097// FastEmit functions for X86ISD::PROBED_ALLOCA.
5098
5099unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5100 if (RetVT.SimpleTy != MVT::i32)
5101 return 0;
5102 if ((!Subtarget->isTarget64BitLP64())) {
5103 return fastEmitInst_r(X86::PROBED_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
5104 }
5105 return 0;
5106}
5107
5108unsigned fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5109 if (RetVT.SimpleTy != MVT::i64)
5110 return 0;
5111 if ((Subtarget->is64Bit())) {
5112 return fastEmitInst_r(X86::PROBED_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
5113 }
5114 return 0;
5115}
5116
5117unsigned fastEmit_X86ISD_PROBED_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5118 switch (VT.SimpleTy) {
5119 case MVT::i32: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
5120 case MVT::i64: return fastEmit_X86ISD_PROBED_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
5121 default: return 0;
5122 }
5123}
5124
5125// FastEmit functions for X86ISD::RCP14.
5126
5127unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5128 if (RetVT.SimpleTy != MVT::v4f32)
5129 return 0;
5130 if ((Subtarget->hasVLX())) {
5131 return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
5132 }
5133 return 0;
5134}
5135
5136unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5137 if (RetVT.SimpleTy != MVT::v8f32)
5138 return 0;
5139 if ((Subtarget->hasVLX())) {
5140 return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
5141 }
5142 return 0;
5143}
5144
5145unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5146 if (RetVT.SimpleTy != MVT::v16f32)
5147 return 0;
5148 if ((Subtarget->hasAVX512())) {
5149 return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
5150 }
5151 return 0;
5152}
5153
5154unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5155 if (RetVT.SimpleTy != MVT::v2f64)
5156 return 0;
5157 if ((Subtarget->hasVLX())) {
5158 return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
5159 }
5160 return 0;
5161}
5162
5163unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5164 if (RetVT.SimpleTy != MVT::v4f64)
5165 return 0;
5166 if ((Subtarget->hasVLX())) {
5167 return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
5168 }
5169 return 0;
5170}
5171
5172unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5173 if (RetVT.SimpleTy != MVT::v8f64)
5174 return 0;
5175 if ((Subtarget->hasAVX512())) {
5176 return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
5177 }
5178 return 0;
5179}
5180
5181unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5182 switch (VT.SimpleTy) {
5183 case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
5184 case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
5185 case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5186 case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5187 case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
5188 case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5189 default: return 0;
5190 }
5191}
5192
5193// FastEmit functions for X86ISD::RCP28.
5194
5195unsigned fastEmit_X86ISD_RCP28_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5196 if (RetVT.SimpleTy != MVT::v16f32)
5197 return 0;
5198 if ((Subtarget->hasERI())) {
5199 return fastEmitInst_r(X86::VRCP28PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
5200 }
5201 return 0;
5202}
5203
5204unsigned fastEmit_X86ISD_RCP28_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5205 if (RetVT.SimpleTy != MVT::v8f64)
5206 return 0;
5207 if ((Subtarget->hasERI())) {
5208 return fastEmitInst_r(X86::VRCP28PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
5209 }
5210 return 0;
5211}
5212
5213unsigned fastEmit_X86ISD_RCP28_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5214 switch (VT.SimpleTy) {
5215 case MVT::v16f32: return fastEmit_X86ISD_RCP28_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5216 case MVT::v8f64: return fastEmit_X86ISD_RCP28_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5217 default: return 0;
5218 }
5219}
5220
5221// FastEmit functions for X86ISD::RCP28_SAE.
5222
5223unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5224 if (RetVT.SimpleTy != MVT::v16f32)
5225 return 0;
5226 if ((Subtarget->hasERI())) {
5227 return fastEmitInst_r(X86::VRCP28PSZrb, &X86::VR512RegClass, Op0, Op0IsKill);
5228 }
5229 return 0;
5230}
5231
5232unsigned fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5233 if (RetVT.SimpleTy != MVT::v8f64)
5234 return 0;
5235 if ((Subtarget->hasERI())) {
5236 return fastEmitInst_r(X86::VRCP28PDZrb, &X86::VR512RegClass, Op0, Op0IsKill);
5237 }
5238 return 0;
5239}
5240
5241unsigned fastEmit_X86ISD_RCP28_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5242 switch (VT.SimpleTy) {
5243 case MVT::v16f32: return fastEmit_X86ISD_RCP28_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5244 case MVT::v8f64: return fastEmit_X86ISD_RCP28_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5245 default: return 0;
5246 }
5247}
5248
5249// FastEmit functions for X86ISD::RSQRT14.
5250
5251unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5252 if (RetVT.SimpleTy != MVT::v4f32)
5253 return 0;
5254 if ((Subtarget->hasVLX())) {
5255 return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
5256 }
5257 return 0;
5258}
5259
5260unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5261 if (RetVT.SimpleTy != MVT::v8f32)
5262 return 0;
5263 if ((Subtarget->hasVLX())) {
5264 return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
5265 }
5266 return 0;
5267}
5268
5269unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5270 if (RetVT.SimpleTy != MVT::v16f32)
5271 return 0;
5272 if ((Subtarget->hasAVX512())) {
5273 return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
5274 }
5275 return 0;
5276}
5277
5278unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5279 if (RetVT.SimpleTy != MVT::v2f64)
5280 return 0;
5281 if ((Subtarget->hasVLX())) {
5282 return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
5283 }
5284 return 0;
5285}
5286
5287unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5288 if (RetVT.SimpleTy != MVT::v4f64)
5289 return 0;
5290 if ((Subtarget->hasVLX())) {
5291 return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
5292 }
5293 return 0;
5294}
5295
5296unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5297 if (RetVT.SimpleTy != MVT::v8f64)
5298 return 0;
5299 if ((Subtarget->hasAVX512())) {
5300 return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
5301 }
5302 return 0;
5303}
5304
5305unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5306 switch (VT.SimpleTy) {
5307 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
5308 case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
5309 case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5310 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5311 case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
5312 case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5313 default: return 0;
5314 }
5315}
5316
5317// FastEmit functions for X86ISD::RSQRT28.
5318
5319unsigned fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5320 if (RetVT.SimpleTy != MVT::v16f32)
5321 return 0;
5322 if ((Subtarget->hasERI())) {
5323 return fastEmitInst_r(X86::VRSQRT28PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
5324 }
5325 return 0;
5326}
5327
5328unsigned fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5329 if (RetVT.SimpleTy != MVT::v8f64)
5330 return 0;
5331 if ((Subtarget->hasERI())) {
5332 return fastEmitInst_r(X86::VRSQRT28PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
5333 }
5334 return 0;
5335}
5336
5337unsigned fastEmit_X86ISD_RSQRT28_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5338 switch (VT.SimpleTy) {
5339 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5340 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5341 default: return 0;
5342 }
5343}
5344
5345// FastEmit functions for X86ISD::RSQRT28_SAE.
5346
5347unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5348 if (RetVT.SimpleTy != MVT::v16f32)
5349 return 0;
5350 if ((Subtarget->hasERI())) {
5351 return fastEmitInst_r(X86::VRSQRT28PSZrb, &X86::VR512RegClass, Op0, Op0IsKill);
5352 }
5353 return 0;
5354}
5355
5356unsigned fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5357 if (RetVT.SimpleTy != MVT::v8f64)
5358 return 0;
5359 if ((Subtarget->hasERI())) {
5360 return fastEmitInst_r(X86::VRSQRT28PDZrb, &X86::VR512RegClass, Op0, Op0IsKill);
5361 }
5362 return 0;
5363}
5364
5365unsigned fastEmit_X86ISD_RSQRT28_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5366 switch (VT.SimpleTy) {
5367 case MVT::v16f32: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5368 case MVT::v8f64: return fastEmit_X86ISD_RSQRT28_SAE_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5369 default: return 0;
5370 }
5371}
5372
5373// FastEmit functions for X86ISD::SEG_ALLOCA.
5374
5375unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5376 if (RetVT.SimpleTy != MVT::i32)
5377 return 0;
5378 if ((!Subtarget->isTarget64BitLP64())) {
5379 return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
5380 }
5381 return 0;
5382}
5383
5384unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5385 if (RetVT.SimpleTy != MVT::i64)
5386 return 0;
5387 if ((Subtarget->is64Bit())) {
5388 return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
5389 }
5390 return 0;
5391}
5392
5393unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5394 switch (VT.SimpleTy) {
5395 case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
5396 case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
5397 default: return 0;
5398 }
5399}
5400
5401// FastEmit functions for X86ISD::STRICT_CVTPH2PS.
5402
5403unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
5404 if ((Subtarget->hasVLX())) {
5405 return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5406 }
5407 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
5408 return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
5409 }
5410 return 0;
5411}
5412
5413unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
5414 if ((Subtarget->hasVLX())) {
5415 return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5416 }
5417 if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
5418 return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5419 }
5420 return 0;
5421}
5422
5423unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5424switch (RetVT.SimpleTy) {
5425 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
5426 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
5427 default: return 0;
5428}
5429}
5430
5431unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5432 if (RetVT.SimpleTy != MVT::v16f32)
5433 return 0;
5434 if ((Subtarget->hasAVX512())) {
5435 return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5436 }
5437 return 0;
5438}
5439
5440unsigned fastEmit_X86ISD_STRICT_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5441 switch (VT.SimpleTy) {
5442 case MVT::v8i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
5443 case MVT::v16i16: return fastEmit_X86ISD_STRICT_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
5444 default: return 0;
5445 }
5446}
5447
5448// FastEmit functions for X86ISD::STRICT_CVTSI2P.
5449
5450unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5451 if (RetVT.SimpleTy != MVT::v2f64)
5452 return 0;
5453 if ((Subtarget->hasVLX())) {
5454 return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5455 }
5456 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5457 return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
5458 }
5459 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5460 return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
5461 }
5462 return 0;
5463}
5464
5465unsigned fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5466 if (RetVT.SimpleTy != MVT::v4f32)
5467 return 0;
5468 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5469 return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5470 }
5471 return 0;
5472}
5473
5474unsigned fastEmit_X86ISD_STRICT_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5475 switch (VT.SimpleTy) {
5476 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5477 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5478 default: return 0;
5479 }
5480}
5481
5482// FastEmit functions for X86ISD::STRICT_CVTTP2SI.
5483
5484unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5485 if ((Subtarget->hasVLX())) {
5486 return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5487 }
5488 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5489 return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
5490 }
5491 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5492 return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
5493 }
5494 return 0;
5495}
5496
5497unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
5498 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5499 return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5500 }
5501 return 0;
5502}
5503
5504unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5505 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5506 return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5507 }
5508 return 0;
5509}
5510
5511unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5512switch (RetVT.SimpleTy) {
5513 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
5514 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
5515 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
5516 default: return 0;
5517}
5518}
5519
5520unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5521 if ((Subtarget->hasVLX())) {
5522 return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5523 }
5524 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5525 return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5526 }
5527 return 0;
5528}
5529
5530unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5531 if ((Subtarget->hasDQI())) {
5532 return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5533 }
5534 return 0;
5535}
5536
5537unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5538switch (RetVT.SimpleTy) {
5539 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
5540 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
5541 default: return 0;
5542}
5543}
5544
5545unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5546 if (RetVT.SimpleTy != MVT::v16i32)
5547 return 0;
5548 if ((Subtarget->hasAVX512())) {
5549 return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5550 }
5551 return 0;
5552}
5553
5554unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5555 if ((Subtarget->hasVLX())) {
5556 return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5557 }
5558 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5559 return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
5560 }
5561 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5562 return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
5563 }
5564 return 0;
5565}
5566
5567unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
5568 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5569 return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5570 }
5571 return 0;
5572}
5573
5574unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5575switch (RetVT.SimpleTy) {
5576 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
5577 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
5578 default: return 0;
5579}
5580}
5581
5582unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5583 if ((Subtarget->hasVLX())) {
5584 return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5585 }
5586 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5587 return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
5588 }
5589 return 0;
5590}
5591
5592unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5593 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5594 return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5595 }
5596 return 0;
5597}
5598
5599unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5600switch (RetVT.SimpleTy) {
5601 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
5602 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
5603 default: return 0;
5604}
5605}
5606
5607unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5608 if ((Subtarget->hasAVX512())) {
5609 return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5610 }
5611 return 0;
5612}
5613
5614unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5615 if ((Subtarget->hasDQI())) {
5616 return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5617 }
5618 return 0;
5619}
5620
5621unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5622switch (RetVT.SimpleTy) {
5623 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
5624 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
5625 default: return 0;
5626}
5627}
5628
5629unsigned fastEmit_X86ISD_STRICT_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5630 switch (VT.SimpleTy) {
5631 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
5632 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
5633 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5634 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5635 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
5636 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5637 default: return 0;
5638 }
5639}
5640
5641// FastEmit functions for X86ISD::STRICT_CVTTP2UI.
5642
5643unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5644 if ((Subtarget->hasVLX())) {
5645 return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5646 }
5647 return 0;
5648}
5649
5650unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
5651 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5652 return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5653 }
5654 return 0;
5655}
5656
5657unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5658 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5659 return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5660 }
5661 return 0;
5662}
5663
5664unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5665switch (RetVT.SimpleTy) {
5666 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
5667 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
5668 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
5669 default: return 0;
5670}
5671}
5672
5673unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5674 if ((Subtarget->hasVLX())) {
5675 return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5676 }
5677 return 0;
5678}
5679
5680unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5681 if ((Subtarget->hasDQI())) {
5682 return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5683 }
5684 return 0;
5685}
5686
5687unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5688switch (RetVT.SimpleTy) {
5689 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
5690 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
5691 default: return 0;
5692}
5693}
5694
5695unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5696 if (RetVT.SimpleTy != MVT::v16i32)
5697 return 0;
5698 if ((Subtarget->hasAVX512())) {
5699 return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5700 }
5701 return 0;
5702}
5703
5704unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5705 if ((Subtarget->hasVLX())) {
5706 return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5707 }
5708 return 0;
5709}
5710
5711unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
5712 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5713 return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5714 }
5715 return 0;
5716}
5717
5718unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5719switch (RetVT.SimpleTy) {
5720 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
5721 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
5722 default: return 0;
5723}
5724}
5725
5726unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5727 if ((Subtarget->hasVLX())) {
5728 return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5729 }
5730 return 0;
5731}
5732
5733unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5734 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5735 return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5736 }
5737 return 0;
5738}
5739
5740unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5741switch (RetVT.SimpleTy) {
5742 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
5743 case MVT::v4i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
5744 default: return 0;
5745}
5746}
5747
5748unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5749 if ((Subtarget->hasAVX512())) {
5750 return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5751 }
5752 return 0;
5753}
5754
5755unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5756 if ((Subtarget->hasDQI())) {
5757 return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5758 }
5759 return 0;
5760}
5761
5762unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5763switch (RetVT.SimpleTy) {
5764 case MVT::v8i32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
5765 case MVT::v8i64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
5766 default: return 0;
5767}
5768}
5769
5770unsigned fastEmit_X86ISD_STRICT_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5771 switch (VT.SimpleTy) {
5772 case MVT::v4f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
5773 case MVT::v8f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
5774 case MVT::v16f32: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
5775 case MVT::v2f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5776 case MVT::v4f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
5777 case MVT::v8f64: return fastEmit_X86ISD_STRICT_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5778 default: return 0;
5779 }
5780}
5781
5782// FastEmit functions for X86ISD::STRICT_CVTUI2P.
5783
5784unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5785 if (RetVT.SimpleTy != MVT::v2f64)
5786 return 0;
5787 if ((Subtarget->hasVLX())) {
5788 return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5789 }
5790 return 0;
5791}
5792
5793unsigned fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5794 if (RetVT.SimpleTy != MVT::v4f32)
5795 return 0;
5796 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
5797 return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5798 }
5799 return 0;
5800}
5801
5802unsigned fastEmit_X86ISD_STRICT_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5803 switch (VT.SimpleTy) {
5804 case MVT::v4i32: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
5805 case MVT::v2i64: return fastEmit_X86ISD_STRICT_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
5806 default: return 0;
5807 }
5808}
5809
5810// FastEmit functions for X86ISD::STRICT_VFPEXT.
5811
5812unsigned fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5813 if (RetVT.SimpleTy != MVT::v2f64)
5814 return 0;
5815 if ((Subtarget->hasVLX())) {
5816 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5817 }
5818 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5819 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
5820 }
5821 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5822 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
5823 }
5824 return 0;
5825}
5826
5827unsigned fastEmit_X86ISD_STRICT_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5828 switch (VT.SimpleTy) {
5829 case MVT::v4f32: return fastEmit_X86ISD_STRICT_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
5830 default: return 0;
5831 }
5832}
5833
5834// FastEmit functions for X86ISD::STRICT_VFPROUND.
5835
5836unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5837 if (RetVT.SimpleTy != MVT::v4f32)
5838 return 0;
5839 if ((Subtarget->hasVLX())) {
5840 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5841 }
5842 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
5843 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
5844 }
5845 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5846 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
5847 }
5848 return 0;
5849}
5850
5851unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5852 if (RetVT.SimpleTy != MVT::v4f32)
5853 return 0;
5854 if ((Subtarget->hasVLX())) {
5855 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5856 }
5857 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
5858 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
5859 }
5860 return 0;
5861}
5862
5863unsigned fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5864 if (RetVT.SimpleTy != MVT::v8f32)
5865 return 0;
5866 if ((Subtarget->hasAVX512())) {
5867 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
5868 }
5869 return 0;
5870}
5871
5872unsigned fastEmit_X86ISD_STRICT_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
5873 switch (VT.SimpleTy) {
5874 case MVT::v2f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
5875 case MVT::v4f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
5876 case MVT::v8f64: return fastEmit_X86ISD_STRICT_VFPROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
5877 default: return 0;
5878 }
5879}
5880
5881// FastEmit functions for X86ISD::VBROADCAST.
5882
5883unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
5884 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5885 return fastEmitInst_r(X86::VPBROADCASTDrZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5886 }
5887 return 0;
5888}
5889
5890unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
5891 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5892 return fastEmitInst_r(X86::VPBROADCASTDrZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5893 }
5894 return 0;
5895}
5896
5897unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
5898 if ((Subtarget->hasAVX512())) {
5899 return fastEmitInst_r(X86::VPBROADCASTDrZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5900 }
5901 return 0;
5902}
5903
5904unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5905switch (RetVT.SimpleTy) {
5906 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
5907 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
5908 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
5909 default: return 0;
5910}
5911}
5912
5913unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
5914 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5915 return fastEmitInst_r(X86::VPBROADCASTQrZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5916 }
5917 return 0;
5918}
5919
5920unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
5921 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
5922 return fastEmitInst_r(X86::VPBROADCASTQrZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5923 }
5924 return 0;
5925}
5926
5927unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
5928 if ((Subtarget->hasAVX512())) {
5929 return fastEmitInst_r(X86::VPBROADCASTQrZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5930 }
5931 return 0;
5932}
5933
5934unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5935switch (RetVT.SimpleTy) {
5936 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
5937 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
5938 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
5939 default: return 0;
5940}
5941}
5942
5943unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
5944 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5945 return fastEmitInst_r(X86::VPBROADCASTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5946 }
5947 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5948 return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
5949 }
5950 return 0;
5951}
5952
5953unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
5954 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5955 return fastEmitInst_r(X86::VPBROADCASTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5956 }
5957 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5958 return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5959 }
5960 return 0;
5961}
5962
5963unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
5964 if ((Subtarget->hasBWI())) {
5965 return fastEmitInst_r(X86::VPBROADCASTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
5966 }
5967 return 0;
5968}
5969
5970unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
5971switch (RetVT.SimpleTy) {
5972 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
5973 case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
5974 case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
5975 default: return 0;
5976}
5977}
5978
5979unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
5980 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5981 return fastEmitInst_r(X86::VPBROADCASTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
5982 }
5983 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5984 return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
5985 }
5986 return 0;
5987}
5988
5989unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
5990 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
5991 return fastEmitInst_r(X86::VPBROADCASTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
5992 }
5993 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
5994 return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
5995 }
5996 return 0;
5997}
5998
5999unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
6000 if ((Subtarget->hasBWI())) {
6001 return fastEmitInst_r(X86::VPBROADCASTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6002 }
6003 return 0;
6004}
6005
6006unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6007switch (RetVT.SimpleTy) {
6008 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
6009 case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
6010 case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
6011 default: return 0;
6012}
6013}
6014
6015unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6016 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6017 return fastEmitInst_r(X86::VPBROADCASTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6018 }
6019 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6020 return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
6021 }
6022 return 0;
6023}
6024
6025unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
6026 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6027 return fastEmitInst_r(X86::VPBROADCASTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6028 }
6029 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6030 return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
6031 }
6032 return 0;
6033}
6034
6035unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
6036 if ((Subtarget->hasAVX512())) {
6037 return fastEmitInst_r(X86::VPBROADCASTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6038 }
6039 return 0;
6040}
6041
6042unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6043switch (RetVT.SimpleTy) {
6044 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
6045 case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
6046 case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
6047 default: return 0;
6048}
6049}
6050
6051unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
6052 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6053 return fastEmitInst_r(X86::VPBROADCASTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6054 }
6055 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6056 return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
6057 }
6058 return 0;
6059}
6060
6061unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
6062 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6063 return fastEmitInst_r(X86::VPBROADCASTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6064 }
6065 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6066 return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
6067 }
6068 return 0;
6069}
6070
6071unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
6072 if ((Subtarget->hasAVX512())) {
6073 return fastEmitInst_r(X86::VPBROADCASTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6074 }
6075 return 0;
6076}
6077
6078unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6079switch (RetVT.SimpleTy) {
6080 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
6081 case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
6082 case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
6083 default: return 0;
6084}
6085}
6086
6087unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
6088 if ((Subtarget->hasVLX())) {
6089 return fastEmitInst_r(X86::VBROADCASTSSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6090 }
6091 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6092 return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
6093 }
6094 return 0;
6095}
6096
6097unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
6098 if ((Subtarget->hasVLX())) {
6099 return fastEmitInst_r(X86::VBROADCASTSSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6100 }
6101 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6102 return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
6103 }
6104 return 0;
6105}
6106
6107unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
6108 if ((Subtarget->hasAVX512())) {
6109 return fastEmitInst_r(X86::VBROADCASTSSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6110 }
6111 return 0;
6112}
6113
6114unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6115switch (RetVT.SimpleTy) {
6116 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
6117 case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
6118 case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
6119 default: return 0;
6120}
6121}
6122
6123unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
6124 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6125 return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
6126 }
6127 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6128 return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6129 }
6130 return 0;
6131}
6132
6133unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
6134 if ((Subtarget->hasVLX())) {
6135 return fastEmitInst_r(X86::VBROADCASTSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6136 }
6137 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
6138 return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
6139 }
6140 return 0;
6141}
6142
6143unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
6144 if ((Subtarget->hasAVX512())) {
6145 return fastEmitInst_r(X86::VBROADCASTSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6146 }
6147 return 0;
6148}
6149
6150unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6151switch (RetVT.SimpleTy) {
6152 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
6153 case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
6154 case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
6155 default: return 0;
6156}
6157}
6158
6159unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6160 switch (VT.SimpleTy) {
6161 case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
6162 case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
6163 case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
6164 case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
6165 case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
6166 case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
6167 case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
6168 case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
6169 default: return 0;
6170 }
6171}
6172
6173// FastEmit functions for X86ISD::VBROADCASTM.
6174
6175unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
6176 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
6177 return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6178 }
6179 return 0;
6180}
6181
6182unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
6183 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
6184 return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6185 }
6186 return 0;
6187}
6188
6189unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
6190 if ((Subtarget->hasCDI())) {
6191 return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6192 }
6193 return 0;
6194}
6195
6196unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6197switch (RetVT.SimpleTy) {
6198 case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
6199 case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
6200 case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
6201 default: return 0;
6202}
6203}
6204
6205unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6206 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
6207 return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6208 }
6209 return 0;
6210}
6211
6212unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
6213 if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
6214 return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
6215 }
6216 return 0;
6217}
6218
6219unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
6220 if ((Subtarget->hasCDI())) {
6221 return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
6222 }
6223 return 0;
6224}
6225
6226unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6227switch (RetVT.SimpleTy) {
6228 case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
6229 case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
6230 case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
6231 default: return 0;
6232}
6233}
6234
6235unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6236 switch (VT.SimpleTy) {
6237 case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
6238 case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
6239 default: return 0;
6240 }
6241}
6242
6243// FastEmit functions for X86ISD::VFPEXT.
6244
6245unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6246 if (RetVT.SimpleTy != MVT::v2f64)
6247 return 0;
6248 if ((Subtarget->hasVLX())) {
6249 return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6250 }
6251 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6252 return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
6253 }
6254 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6255 return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
6256 }
6257 return 0;
6258}
6259
6260unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6261 switch (VT.SimpleTy) {
6262 case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
6263 default: return 0;
6264 }
6265}
6266
6267// FastEmit functions for X86ISD::VFPEXT_SAE.
6268
6269unsigned fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6270 if (RetVT.SimpleTy != MVT::v8f64)
6271 return 0;
6272 if ((Subtarget->hasAVX512())) {
6273 return fastEmitInst_r(X86::VCVTPS2PDZrrb, &X86::VR512RegClass, Op0, Op0IsKill);
6274 }
6275 return 0;
6276}
6277
6278unsigned fastEmit_X86ISD_VFPEXT_SAE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6279 switch (VT.SimpleTy) {
6280 case MVT::v8f32: return fastEmit_X86ISD_VFPEXT_SAE_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
6281 default: return 0;
6282 }
6283}
6284
6285// FastEmit functions for X86ISD::VFPROUND.
6286
6287unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6288 if (RetVT.SimpleTy != MVT::v4f32)
6289 return 0;
6290 if ((Subtarget->hasVLX())) {
6291 return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6292 }
6293 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6294 return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
6295 }
6296 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6297 return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
6298 }
6299 return 0;
6300}
6301
6302unsigned fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6303 if (RetVT.SimpleTy != MVT::v4f32)
6304 return 0;
6305 if ((Subtarget->hasVLX())) {
6306 return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6307 }
6308 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
6309 return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
6310 }
6311 return 0;
6312}
6313
6314unsigned fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6315 if (RetVT.SimpleTy != MVT::v8f32)
6316 return 0;
6317 if ((Subtarget->hasAVX512())) {
6318 return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6319 }
6320 return 0;
6321}
6322
6323unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6324 switch (VT.SimpleTy) {
6325 case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
6326 case MVT::v4f64: return fastEmit_X86ISD_VFPROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
6327 case MVT::v8f64: return fastEmit_X86ISD_VFPROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
6328 default: return 0;
6329 }
6330}
6331
6332// FastEmit functions for X86ISD::VTRUNC.
6333
6334unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6335 if (RetVT.SimpleTy != MVT::v16i8)
6336 return 0;
6337 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6338 return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6339 }
6340 return 0;
6341}
6342
6343unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6344 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6345 return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6346 }
6347 return 0;
6348}
6349
6350unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6351 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6352 return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6353 }
6354 return 0;
6355}
6356
6357unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6358switch (RetVT.SimpleTy) {
6359 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
6360 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
6361 default: return 0;
6362}
6363}
6364
6365unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6366 if (RetVT.SimpleTy != MVT::v16i8)
6367 return 0;
6368 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6369 return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6370 }
6371 return 0;
6372}
6373
6374unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6375 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6376 return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6377 }
6378 return 0;
6379}
6380
6381unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6382 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6383 return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6384 }
6385 return 0;
6386}
6387
6388unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6389 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6390 return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6391 }
6392 return 0;
6393}
6394
6395unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6396switch (RetVT.SimpleTy) {
6397 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
6398 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
6399 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
6400 default: return 0;
6401}
6402}
6403
6404unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6405 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6406 return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6407 }
6408 return 0;
6409}
6410
6411unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6412 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6413 return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6414 }
6415 return 0;
6416}
6417
6418unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6419switch (RetVT.SimpleTy) {
6420 case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
6421 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
6422 default: return 0;
6423}
6424}
6425
6426unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6427 if (RetVT.SimpleTy != MVT::v16i8)
6428 return 0;
6429 if ((Subtarget->hasAVX512())) {
6430 return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6431 }
6432 return 0;
6433}
6434
6435unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6436 switch (VT.SimpleTy) {
6437 case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
6438 case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
6439 case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
6440 case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
6441 case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
6442 case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
6443 default: return 0;
6444 }
6445}
6446
6447// FastEmit functions for X86ISD::VTRUNCS.
6448
6449unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6450 if (RetVT.SimpleTy != MVT::v16i8)
6451 return 0;
6452 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6453 return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6454 }
6455 return 0;
6456}
6457
6458unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6459 if (RetVT.SimpleTy != MVT::v16i8)
6460 return 0;
6461 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6462 return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6463 }
6464 return 0;
6465}
6466
6467unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6468 if (RetVT.SimpleTy != MVT::v32i8)
6469 return 0;
6470 if ((Subtarget->hasBWI())) {
6471 return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6472 }
6473 return 0;
6474}
6475
6476unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6477 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6478 return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6479 }
6480 return 0;
6481}
6482
6483unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6484 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6485 return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6486 }
6487 return 0;
6488}
6489
6490unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6491switch (RetVT.SimpleTy) {
6492 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
6493 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
6494 default: return 0;
6495}
6496}
6497
6498unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6499 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6500 return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6501 }
6502 return 0;
6503}
6504
6505unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6506 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6507 return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6508 }
6509 return 0;
6510}
6511
6512unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6513switch (RetVT.SimpleTy) {
6514 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
6515 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
6516 default: return 0;
6517}
6518}
6519
6520unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6521 if ((Subtarget->hasAVX512())) {
6522 return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6523 }
6524 return 0;
6525}
6526
6527unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
6528 if ((Subtarget->hasAVX512())) {
6529 return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6530 }
6531 return 0;
6532}
6533
6534unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6535switch (RetVT.SimpleTy) {
6536 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
6537 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
6538 default: return 0;
6539}
6540}
6541
6542unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6543 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6544 return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6545 }
6546 return 0;
6547}
6548
6549unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6550 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6551 return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6552 }
6553 return 0;
6554}
6555
6556unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6557 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6558 return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6559 }
6560 return 0;
6561}
6562
6563unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6564switch (RetVT.SimpleTy) {
6565 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
6566 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
6567 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
6568 default: return 0;
6569}
6570}
6571
6572unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6573 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6574 return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6575 }
6576 return 0;
6577}
6578
6579unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6580 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6581 return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6582 }
6583 return 0;
6584}
6585
6586unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6587 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6588 return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6589 }
6590 return 0;
6591}
6592
6593unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6594switch (RetVT.SimpleTy) {
6595 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
6596 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
6597 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
6598 default: return 0;
6599}
6600}
6601
6602unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6603 if ((Subtarget->hasAVX512())) {
6604 return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6605 }
6606 return 0;
6607}
6608
6609unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6610 if ((Subtarget->hasAVX512())) {
6611 return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6612 }
6613 return 0;
6614}
6615
6616unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
6617 if ((Subtarget->hasAVX512())) {
6618 return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6619 }
6620 return 0;
6621}
6622
6623unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6624switch (RetVT.SimpleTy) {
6625 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
6626 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
6627 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
6628 default: return 0;
6629}
6630}
6631
6632unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6633 switch (VT.SimpleTy) {
6634 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
6635 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
6636 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
6637 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
6638 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
6639 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
6640 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
6641 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
6642 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
6643 default: return 0;
6644 }
6645}
6646
6647// FastEmit functions for X86ISD::VTRUNCUS.
6648
6649unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6650 if (RetVT.SimpleTy != MVT::v16i8)
6651 return 0;
6652 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6653 return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6654 }
6655 return 0;
6656}
6657
6658unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6659 if (RetVT.SimpleTy != MVT::v16i8)
6660 return 0;
6661 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
6662 return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6663 }
6664 return 0;
6665}
6666
6667unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6668 if (RetVT.SimpleTy != MVT::v32i8)
6669 return 0;
6670 if ((Subtarget->hasBWI())) {
6671 return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6672 }
6673 return 0;
6674}
6675
6676unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6677 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6678 return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6679 }
6680 return 0;
6681}
6682
6683unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6684 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6685 return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6686 }
6687 return 0;
6688}
6689
6690unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6691switch (RetVT.SimpleTy) {
6692 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
6693 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
6694 default: return 0;
6695}
6696}
6697
6698unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6699 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6700 return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6701 }
6702 return 0;
6703}
6704
6705unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6706 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6707 return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6708 }
6709 return 0;
6710}
6711
6712unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6713switch (RetVT.SimpleTy) {
6714 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
6715 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
6716 default: return 0;
6717}
6718}
6719
6720unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6721 if ((Subtarget->hasAVX512())) {
6722 return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6723 }
6724 return 0;
6725}
6726
6727unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
6728 if ((Subtarget->hasAVX512())) {
6729 return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6730 }
6731 return 0;
6732}
6733
6734unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6735switch (RetVT.SimpleTy) {
6736 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
6737 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
6738 default: return 0;
6739}
6740}
6741
6742unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6743 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6744 return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6745 }
6746 return 0;
6747}
6748
6749unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6750 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6751 return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6752 }
6753 return 0;
6754}
6755
6756unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6757 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6758 return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6759 }
6760 return 0;
6761}
6762
6763unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6764switch (RetVT.SimpleTy) {
6765 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
6766 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
6767 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
6768 default: return 0;
6769}
6770}
6771
6772unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6773 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6774 return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6775 }
6776 return 0;
6777}
6778
6779unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6780 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6781 return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6782 }
6783 return 0;
6784}
6785
6786unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
6787 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
6788 return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
6789 }
6790 return 0;
6791}
6792
6793unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6794switch (RetVT.SimpleTy) {
6795 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
6796 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
6797 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
6798 default: return 0;
6799}
6800}
6801
6802unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
6803 if ((Subtarget->hasAVX512())) {
6804 return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6805 }
6806 return 0;
6807}
6808
6809unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
6810 if ((Subtarget->hasAVX512())) {
6811 return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6812 }
6813 return 0;
6814}
6815
6816unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
6817 if ((Subtarget->hasAVX512())) {
6818 return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
6819 }
6820 return 0;
6821}
6822
6823unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6824switch (RetVT.SimpleTy) {
6825 case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
6826 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
6827 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
6828 default: return 0;
6829}
6830}
6831
6832unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6833 switch (VT.SimpleTy) {
6834 case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
6835 case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
6836 case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
6837 case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
6838 case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
6839 case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
6840 case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
6841 case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
6842 case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
6843 default: return 0;
6844 }
6845}
6846
6847// FastEmit functions for X86ISD::VZEXT_MOVL.
6848
6849unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6850 if (RetVT.SimpleTy != MVT::v2i64)
6851 return 0;
6852 if ((Subtarget->hasAVX512())) {
6853 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6854 }
6855 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6856 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
6857 }
6858 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6859 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
6860 }
6861 return 0;
6862}
6863
6864unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6865 if (RetVT.SimpleTy != MVT::v2f64)
6866 return 0;
6867 if ((Subtarget->hasAVX512())) {
6868 return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
6869 }
6870 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
6871 return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
6872 }
6873 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
6874 return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
6875 }
6876 return 0;
6877}
6878
6879unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6880 switch (VT.SimpleTy) {
6881 case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
6882 case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
6883 default: return 0;
6884 }
6885}
6886
6887// FastEmit functions for X86ISD::WIN_ALLOCA.
6888
6889unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6890 if (RetVT.SimpleTy != MVT::isVoid)
6891 return 0;
6892 if ((!Subtarget->isTarget64BitLP64())) {
6893 return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
6894 }
6895 return 0;
6896}
6897
6898unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
6899 if (RetVT.SimpleTy != MVT::isVoid)
6900 return 0;
6901 if ((Subtarget->is64Bit())) {
6902 return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
6903 }
6904 return 0;
6905}
6906
6907unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
6908 switch (VT.SimpleTy) {
6909 case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
6910 case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
6911 default: return 0;
6912 }
6913}
6914
6915// Top-level FastEmit function.
6916
6917unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
6918 switch (Opcode) {
6919 case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
6920 case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
6921 case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
6922 case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
6923 case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
6924 case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
6925 case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
6926 case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
6927 case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
6928 case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
6929 case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
6930 case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
6931 case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
6932 case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
6933 case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
6934 case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
6935 case ISD::LLRINT: return fastEmit_ISD_LLRINT_r(VT, RetVT, Op0, Op0IsKill);
6936 case ISD::LRINT: return fastEmit_ISD_LRINT_r(VT, RetVT, Op0, Op0IsKill);
6937 case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
6938 case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
6939 case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
6940 case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
6941 case ISD::STRICT_FP_EXTEND: return fastEmit_ISD_STRICT_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
6942 case ISD::STRICT_FP_ROUND: return fastEmit_ISD_STRICT_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
6943 case ISD::STRICT_FP_TO_SINT: return fastEmit_ISD_STRICT_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
6944 case ISD::STRICT_FP_TO_UINT: return fastEmit_ISD_STRICT_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
6945 case ISD::STRICT_FSQRT: return fastEmit_ISD_STRICT_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
6946 case ISD::STRICT_SINT_TO_FP: return fastEmit_ISD_STRICT_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
6947 case ISD::STRICT_UINT_TO_FP: return fastEmit_ISD_STRICT_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
6948 case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
6949 case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
6950 case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
6951 case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
6952 case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
6953 case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
6954 case X86ISD::CVTNEPS2BF16: return fastEmit_X86ISD_CVTNEPS2BF16_r(VT, RetVT, Op0, Op0IsKill);
6955 case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
6956 case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
6957 case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
6958 case X86ISD::CVTPH2PS_SAE: return fastEmit_X86ISD_CVTPH2PS_SAE_r(VT, RetVT, Op0, Op0IsKill);
6959 case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0, Op0IsKill);
6960 case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0, Op0IsKill);
6961 case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
6962 case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
6963 case X86ISD::CVTTP2SI_SAE: return fastEmit_X86ISD_CVTTP2SI_SAE_r(VT, RetVT, Op0, Op0IsKill);
6964 case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
6965 case X86ISD::CVTTP2UI_SAE: return fastEmit_X86ISD_CVTTP2UI_SAE_r(VT, RetVT, Op0, Op0IsKill);
6966 case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0, Op0IsKill);
6967 case X86ISD::CVTTS2SI_SAE: return fastEmit_X86ISD_CVTTS2SI_SAE_r(VT, RetVT, Op0, Op0IsKill);
6968 case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0, Op0IsKill);
6969 case X86ISD::CVTTS2UI_SAE: return fastEmit_X86ISD_CVTTS2UI_SAE_r(VT, RetVT, Op0, Op0IsKill);
6970 case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
6971 case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
6972 case X86ISD::EXP2: return fastEmit_X86ISD_EXP2_r(VT, RetVT, Op0, Op0IsKill);
6973 case X86ISD::EXP2_SAE: return fastEmit_X86ISD_EXP2_SAE_r(VT, RetVT, Op0, Op0IsKill);
6974 case X86ISD::FGETEXP: return fastEmit_X86ISD_FGETEXP_r(VT, RetVT, Op0, Op0IsKill);
6975 case X86ISD::FGETEXP_SAE: return fastEmit_X86ISD_FGETEXP_SAE_r(VT, RetVT, Op0, Op0IsKill);
6976 case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
6977 case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
6978 case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
6979 case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
6980 case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
6981 case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
6982 case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
6983 case X86ISD::MOVQ2DQ: return fastEmit_X86ISD_MOVQ2DQ_r(VT, RetVT, Op0, Op0IsKill);
6984 case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
6985 case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
6986 case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill);
6987 case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill);
6988 case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
6989 case X86ISD::PROBED_ALLOCA: return fastEmit_X86ISD_PROBED_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
6990 case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
6991 case X86ISD::RCP28: return fastEmit_X86ISD_RCP28_r(VT, RetVT, Op0, Op0IsKill);
6992 case X86ISD::RCP28_SAE: return fastEmit_X86ISD_RCP28_SAE_r(VT, RetVT, Op0, Op0IsKill);
6993 case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
6994 case X86ISD::RSQRT28: return fastEmit_X86ISD_RSQRT28_r(VT, RetVT, Op0, Op0IsKill);
6995 case X86ISD::RSQRT28_SAE: return fastEmit_X86ISD_RSQRT28_SAE_r(VT, RetVT, Op0, Op0IsKill);
6996 case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
6997 case X86ISD::STRICT_CVTPH2PS: return fastEmit_X86ISD_STRICT_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
6998 case X86ISD::STRICT_CVTSI2P: return fastEmit_X86ISD_STRICT_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
6999 case X86ISD::STRICT_CVTTP2SI: return fastEmit_X86ISD_STRICT_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
7000 case X86ISD::STRICT_CVTTP2UI: return fastEmit_X86ISD_STRICT_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
7001 case X86ISD::STRICT_CVTUI2P: return fastEmit_X86ISD_STRICT_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
7002 case X86ISD::STRICT_VFPEXT: return fastEmit_X86ISD_STRICT_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
7003 case X86ISD::STRICT_VFPROUND: return fastEmit_X86ISD_STRICT_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
7004 case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
7005 case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
7006 case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
7007 case X86ISD::VFPEXT_SAE: return fastEmit_X86ISD_VFPEXT_SAE_r(VT, RetVT, Op0, Op0IsKill);
7008 case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
7009 case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
7010 case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
7011 case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
7012 case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
7013 case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
7014 default: return 0;
7015 }
7016}
7017
7018// FastEmit functions for ISD::ADD.
7019
7020unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7021 if (RetVT.SimpleTy != MVT::i8)
7022 return 0;
7023 return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7024}
7025
7026unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7027 if (RetVT.SimpleTy != MVT::i16)
7028 return 0;
7029 return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7030}
7031
7032unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7033 if (RetVT.SimpleTy != MVT::i32)
7034 return 0;
7035 return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7036}
7037
7038unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7039 if (RetVT.SimpleTy != MVT::i64)
7040 return 0;
7041 return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7042}
7043
7044unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7045 if (RetVT.SimpleTy != MVT::v16i8)
7046 return 0;
7047 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7048 return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7049 }
7050 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7051 return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7052 }
7053 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7054 return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7055 }
7056 return 0;
7057}
7058
7059unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7060 if (RetVT.SimpleTy != MVT::v32i8)
7061 return 0;
7062 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7063 return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7064 }
7065 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7066 return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7067 }
7068 return 0;
7069}
7070
7071unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7072 if (RetVT.SimpleTy != MVT::v64i8)
7073 return 0;
7074 if ((Subtarget->hasBWI())) {
7075 return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7076 }
7077 return 0;
7078}
7079
7080unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7081 if (RetVT.SimpleTy != MVT::v8i16)
7082 return 0;
7083 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7084 return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7085 }
7086 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7087 return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7088 }
7089 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7090 return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7091 }
7092 return 0;
7093}
7094
7095unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7096 if (RetVT.SimpleTy != MVT::v16i16)
7097 return 0;
7098 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
7099 return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7100 }
7101 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
7102 return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7103 }
7104 return 0;
7105}
7106
7107unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7108 if (RetVT.SimpleTy != MVT::v32i16)
7109 return 0;
7110 if ((Subtarget->hasBWI())) {
7111 return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7112 }
7113 return 0;
7114}
7115
7116unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7117 if (RetVT.SimpleTy != MVT::v4i32)
7118 return 0;
7119 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7120 return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7121 }
7122 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7123 return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7124 }
7125 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7126 return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7127 }
7128 return 0;
7129}
7130
7131unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7132 if (RetVT.SimpleTy != MVT::v8i32)
7133 return 0;
7134 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7135 return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7136 }
7137 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7138 return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7139 }
7140 return 0;
7141}
7142
7143unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7144 if (RetVT.SimpleTy != MVT::v16i32)
7145 return 0;
7146 if ((Subtarget->hasAVX512())) {
7147 return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7148 }
7149 return 0;
7150}
7151
7152unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7153 if (RetVT.SimpleTy != MVT::v2i64)
7154 return 0;
7155 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7156 return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7157 }
7158 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7159 return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7160 }
7161 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7162 return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7163 }
7164 return 0;
7165}
7166
7167unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7168 if (RetVT.SimpleTy != MVT::v4i64)
7169 return 0;
7170 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7171 return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7172 }
7173 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7174 return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7175 }
7176 return 0;
7177}
7178
7179unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7180 if (RetVT.SimpleTy != MVT::v8i64)
7181 return 0;
7182 if ((Subtarget->hasAVX512())) {
7183 return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7184 }
7185 return 0;
7186}
7187
7188unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7189 switch (VT.SimpleTy) {
7190 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7191 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7192 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7193 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7194 case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7195 case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7196 case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7197 case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7198 case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7199 case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7200 case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7201 case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7202 case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7203 case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7204 case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7205 case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7206 default: return 0;
7207 }
7208}
7209
7210// FastEmit functions for ISD::AND.
7211
7212unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7213 if (RetVT.SimpleTy != MVT::i8)
7214 return 0;
7215 return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7216}
7217
7218unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7219 if (RetVT.SimpleTy != MVT::i16)
7220 return 0;
7221 return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7222}
7223
7224unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7225 if (RetVT.SimpleTy != MVT::i32)
7226 return 0;
7227 return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7228}
7229
7230unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7231 if (RetVT.SimpleTy != MVT::i64)
7232 return 0;
7233 return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7234}
7235
7236unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7237 if (RetVT.SimpleTy != MVT::v8i1)
7238 return 0;
7239 if ((Subtarget->hasDQI())) {
7240 return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7241 }
7242 return 0;
7243}
7244
7245unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7246 if (RetVT.SimpleTy != MVT::v16i1)
7247 return 0;
7248 if ((Subtarget->hasAVX512())) {
7249 return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7250 }
7251 return 0;
7252}
7253
7254unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7255 if (RetVT.SimpleTy != MVT::v32i1)
7256 return 0;
7257 if ((Subtarget->hasBWI())) {
7258 return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7259 }
7260 return 0;
7261}
7262
7263unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7264 if (RetVT.SimpleTy != MVT::v64i1)
7265 return 0;
7266 if ((Subtarget->hasBWI())) {
7267 return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7268 }
7269 return 0;
7270}
7271
7272unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7273 if (RetVT.SimpleTy != MVT::v16i8)
7274 return 0;
7275 if ((Subtarget->hasVLX())) {
7276 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7277 }
7278 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7279 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7280 }
7281 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7282 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7283 }
7284 return 0;
7285}
7286
7287unsigned fastEmit_ISD_AND_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7288 if (RetVT.SimpleTy != MVT::v32i8)
7289 return 0;
7290 if ((Subtarget->hasVLX())) {
7291 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7292 }
7293 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
7294 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7295 }
7296 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7297 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7298 }
7299 return 0;
7300}
7301
7302unsigned fastEmit_ISD_AND_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7303 if (RetVT.SimpleTy != MVT::v64i8)
7304 return 0;
7305 if ((Subtarget->hasAVX512())) {
7306 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7307 }
7308 return 0;
7309}
7310
7311unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7312 if (RetVT.SimpleTy != MVT::v8i16)
7313 return 0;
7314 if ((Subtarget->hasVLX())) {
7315 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7316 }
7317 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7318 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7319 }
7320 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7321 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7322 }
7323 return 0;
7324}
7325
7326unsigned fastEmit_ISD_AND_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7327 if (RetVT.SimpleTy != MVT::v16i16)
7328 return 0;
7329 if ((Subtarget->hasVLX())) {
7330 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7331 }
7332 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
7333 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7334 }
7335 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7336 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7337 }
7338 return 0;
7339}
7340
7341unsigned fastEmit_ISD_AND_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7342 if (RetVT.SimpleTy != MVT::v32i16)
7343 return 0;
7344 if ((Subtarget->hasAVX512())) {
7345 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7346 }
7347 return 0;
7348}
7349
7350unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7351 if (RetVT.SimpleTy != MVT::v4i32)
7352 return 0;
7353 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7354 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7355 }
7356 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7357 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7358 }
7359 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7360 return fastEmitInst_rr(X86::VPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7361 }
7362 return 0;
7363}
7364
7365unsigned fastEmit_ISD_AND_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7366 if (RetVT.SimpleTy != MVT::v8i32)
7367 return 0;
7368 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
7369 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7370 }
7371 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7372 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7373 }
7374 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7375 return fastEmitInst_rr(X86::VPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7376 }
7377 return 0;
7378}
7379
7380unsigned fastEmit_ISD_AND_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7381 if (RetVT.SimpleTy != MVT::v16i32)
7382 return 0;
7383 if ((Subtarget->hasAVX512())) {
7384 return fastEmitInst_rr(X86::VPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7385 }
7386 return 0;
7387}
7388
7389unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7390 if (RetVT.SimpleTy != MVT::v2i64)
7391 return 0;
7392 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7393 return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7394 }
7395 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7396 return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7397 }
7398 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7399 return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7400 }
7401 return 0;
7402}
7403
7404unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7405 if (RetVT.SimpleTy != MVT::v4i64)
7406 return 0;
7407 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
7408 return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7409 }
7410 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7411 return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7412 }
7413 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
7414 return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7415 }
7416 return 0;
7417}
7418
7419unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7420 if (RetVT.SimpleTy != MVT::v8i64)
7421 return 0;
7422 if ((Subtarget->hasAVX512())) {
7423 return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7424 }
7425 return 0;
7426}
7427
7428unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7429 switch (VT.SimpleTy) {
7430 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7431 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7432 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7433 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7434 case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7435 case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7436 case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7437 case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7438 case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7439 case MVT::v32i8: return fastEmit_ISD_AND_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7440 case MVT::v64i8: return fastEmit_ISD_AND_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7441 case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7442 case MVT::v16i16: return fastEmit_ISD_AND_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7443 case MVT::v32i16: return fastEmit_ISD_AND_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7444 case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7445 case MVT::v8i32: return fastEmit_ISD_AND_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7446 case MVT::v16i32: return fastEmit_ISD_AND_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7447 case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7448 case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7449 case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7450 default: return 0;
7451 }
7452}
7453
7454// FastEmit functions for ISD::FADD.
7455
7456unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7457 if (RetVT.SimpleTy != MVT::f32)
7458 return 0;
7459 if ((Subtarget->hasAVX512())) {
7460 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7461 }
7462 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7463 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7464 }
7465 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7466 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7467 }
7468 if ((!Subtarget->hasSSE1())) {
7469 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7470 }
7471 return 0;
7472}
7473
7474unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7475 if (RetVT.SimpleTy != MVT::f64)
7476 return 0;
7477 if ((Subtarget->hasAVX512())) {
7478 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7479 }
7480 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7481 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7482 }
7483 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7484 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7485 }
7486 if ((!Subtarget->hasSSE2())) {
7487 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7488 }
7489 return 0;
7490}
7491
7492unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7493 if (RetVT.SimpleTy != MVT::f80)
7494 return 0;
7495 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7496}
7497
7498unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7499 if (RetVT.SimpleTy != MVT::v4f32)
7500 return 0;
7501 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7502 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7503 }
7504 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7505 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7506 }
7507 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7508 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7509 }
7510 return 0;
7511}
7512
7513unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7514 if (RetVT.SimpleTy != MVT::v8f32)
7515 return 0;
7516 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7517 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7518 }
7519 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7520 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7521 }
7522 return 0;
7523}
7524
7525unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7526 if (RetVT.SimpleTy != MVT::v16f32)
7527 return 0;
7528 if ((Subtarget->hasAVX512())) {
7529 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7530 }
7531 return 0;
7532}
7533
7534unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7535 if (RetVT.SimpleTy != MVT::v2f64)
7536 return 0;
7537 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7538 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7539 }
7540 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7541 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7542 }
7543 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7544 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7545 }
7546 return 0;
7547}
7548
7549unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7550 if (RetVT.SimpleTy != MVT::v4f64)
7551 return 0;
7552 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7553 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7554 }
7555 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7556 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7557 }
7558 return 0;
7559}
7560
7561unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7562 if (RetVT.SimpleTy != MVT::v8f64)
7563 return 0;
7564 if ((Subtarget->hasAVX512())) {
7565 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7566 }
7567 return 0;
7568}
7569
7570unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7571 switch (VT.SimpleTy) {
7572 case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7573 case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7574 case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7575 case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7576 case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7577 case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7578 case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7579 case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7580 case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7581 default: return 0;
7582 }
7583}
7584
7585// FastEmit functions for ISD::FDIV.
7586
7587unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7588 if (RetVT.SimpleTy != MVT::f32)
7589 return 0;
7590 if ((Subtarget->hasAVX512())) {
7591 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7592 }
7593 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7594 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7595 }
7596 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7597 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7598 }
7599 if ((!Subtarget->hasSSE1())) {
7600 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7601 }
7602 return 0;
7603}
7604
7605unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7606 if (RetVT.SimpleTy != MVT::f64)
7607 return 0;
7608 if ((Subtarget->hasAVX512())) {
7609 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7610 }
7611 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7612 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7613 }
7614 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7615 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7616 }
7617 if ((!Subtarget->hasSSE2())) {
7618 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7619 }
7620 return 0;
7621}
7622
7623unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7624 if (RetVT.SimpleTy != MVT::f80)
7625 return 0;
7626 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7627}
7628
7629unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7630 if (RetVT.SimpleTy != MVT::v4f32)
7631 return 0;
7632 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7633 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7634 }
7635 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7636 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7637 }
7638 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7639 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7640 }
7641 return 0;
7642}
7643
7644unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7645 if (RetVT.SimpleTy != MVT::v8f32)
7646 return 0;
7647 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7648 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7649 }
7650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7651 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7652 }
7653 return 0;
7654}
7655
7656unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7657 if (RetVT.SimpleTy != MVT::v16f32)
7658 return 0;
7659 if ((Subtarget->hasAVX512())) {
7660 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7661 }
7662 return 0;
7663}
7664
7665unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7666 if (RetVT.SimpleTy != MVT::v2f64)
7667 return 0;
7668 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7669 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7670 }
7671 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7672 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7673 }
7674 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7675 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7676 }
7677 return 0;
7678}
7679
7680unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7681 if (RetVT.SimpleTy != MVT::v4f64)
7682 return 0;
7683 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7684 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7685 }
7686 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7687 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7688 }
7689 return 0;
7690}
7691
7692unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7693 if (RetVT.SimpleTy != MVT::v8f64)
7694 return 0;
7695 if ((Subtarget->hasAVX512())) {
7696 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7697 }
7698 return 0;
7699}
7700
7701unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7702 switch (VT.SimpleTy) {
7703 case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7704 case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7705 case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7706 case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7707 case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7708 case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7709 case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7710 case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7711 case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7712 default: return 0;
7713 }
7714}
7715
7716// FastEmit functions for ISD::FMUL.
7717
7718unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7719 if (RetVT.SimpleTy != MVT::f32)
7720 return 0;
7721 if ((Subtarget->hasAVX512())) {
7722 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7723 }
7724 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7725 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7726 }
7727 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7728 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7729 }
7730 if ((!Subtarget->hasSSE1())) {
7731 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7732 }
7733 return 0;
7734}
7735
7736unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7737 if (RetVT.SimpleTy != MVT::f64)
7738 return 0;
7739 if ((Subtarget->hasAVX512())) {
7740 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7741 }
7742 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7743 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7744 }
7745 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7746 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7747 }
7748 if ((!Subtarget->hasSSE2())) {
7749 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7750 }
7751 return 0;
7752}
7753
7754unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7755 if (RetVT.SimpleTy != MVT::f80)
7756 return 0;
7757 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7758}
7759
7760unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7761 if (RetVT.SimpleTy != MVT::v4f32)
7762 return 0;
7763 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7764 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7765 }
7766 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7767 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7768 }
7769 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7770 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7771 }
7772 return 0;
7773}
7774
7775unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7776 if (RetVT.SimpleTy != MVT::v8f32)
7777 return 0;
7778 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7779 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7780 }
7781 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7782 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7783 }
7784 return 0;
7785}
7786
7787unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7788 if (RetVT.SimpleTy != MVT::v16f32)
7789 return 0;
7790 if ((Subtarget->hasAVX512())) {
7791 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7792 }
7793 return 0;
7794}
7795
7796unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7797 if (RetVT.SimpleTy != MVT::v2f64)
7798 return 0;
7799 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7800 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7801 }
7802 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7803 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7804 }
7805 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7806 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7807 }
7808 return 0;
7809}
7810
7811unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7812 if (RetVT.SimpleTy != MVT::v4f64)
7813 return 0;
7814 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7815 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7816 }
7817 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7818 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7819 }
7820 return 0;
7821}
7822
7823unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7824 if (RetVT.SimpleTy != MVT::v8f64)
7825 return 0;
7826 if ((Subtarget->hasAVX512())) {
7827 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7828 }
7829 return 0;
7830}
7831
7832unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7833 switch (VT.SimpleTy) {
7834 case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7835 case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7836 case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7837 case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7838 case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7839 case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7840 case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7841 case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7842 case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7843 default: return 0;
7844 }
7845}
7846
7847// FastEmit functions for ISD::FSUB.
7848
7849unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7850 if (RetVT.SimpleTy != MVT::f32)
7851 return 0;
7852 if ((Subtarget->hasAVX512())) {
7853 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7854 }
7855 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7856 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7857 }
7858 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7859 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7860 }
7861 if ((!Subtarget->hasSSE1())) {
7862 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7863 }
7864 return 0;
7865}
7866
7867unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7868 if (RetVT.SimpleTy != MVT::f64)
7869 return 0;
7870 if ((Subtarget->hasAVX512())) {
7871 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7872 }
7873 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7874 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7875 }
7876 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
7877 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7878 }
7879 if ((!Subtarget->hasSSE2())) {
7880 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7881 }
7882 return 0;
7883}
7884
7885unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7886 if (RetVT.SimpleTy != MVT::f80)
7887 return 0;
7888 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7889}
7890
7891unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7892 if (RetVT.SimpleTy != MVT::v4f32)
7893 return 0;
7894 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7895 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7896 }
7897 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
7898 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7899 }
7900 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7901 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7902 }
7903 return 0;
7904}
7905
7906unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7907 if (RetVT.SimpleTy != MVT::v8f32)
7908 return 0;
7909 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7910 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7911 }
7912 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7913 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7914 }
7915 return 0;
7916}
7917
7918unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7919 if (RetVT.SimpleTy != MVT::v16f32)
7920 return 0;
7921 if ((Subtarget->hasAVX512())) {
7922 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7923 }
7924 return 0;
7925}
7926
7927unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7928 if (RetVT.SimpleTy != MVT::v2f64)
7929 return 0;
7930 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7931 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7932 }
7933 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
7934 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7935 }
7936 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7937 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7938 }
7939 return 0;
7940}
7941
7942unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7943 if (RetVT.SimpleTy != MVT::v4f64)
7944 return 0;
7945 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
7946 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7947 }
7948 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
7949 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7950 }
7951 return 0;
7952}
7953
7954unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7955 if (RetVT.SimpleTy != MVT::v8f64)
7956 return 0;
7957 if ((Subtarget->hasAVX512())) {
7958 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7959 }
7960 return 0;
7961}
7962
7963unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7964 switch (VT.SimpleTy) {
7965 case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7966 case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7967 case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7968 case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7969 case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7970 case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7971 case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7972 case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7973 case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
7974 default: return 0;
7975 }
7976}
7977
7978// FastEmit functions for ISD::MUL.
7979
7980unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7981 if (RetVT.SimpleTy != MVT::i8)
7982 return 0;
7983 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
7984 return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
7985}
7986
7987unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7988 if (RetVT.SimpleTy != MVT::i16)
7989 return 0;
7990 return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7991}
7992
7993unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
7994 if (RetVT.SimpleTy != MVT::i32)
7995 return 0;
7996 return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
7997}
7998
7999unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8000 if (RetVT.SimpleTy != MVT::i64)
8001 return 0;
8002 return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8003}
8004
8005unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8006 if (RetVT.SimpleTy != MVT::v8i16)
8007 return 0;
8008 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8009 return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8010 }
8011 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8012 return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8013 }
8014 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8015 return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8016 }
8017 return 0;
8018}
8019
8020unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8021 if (RetVT.SimpleTy != MVT::v16i16)
8022 return 0;
8023 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8024 return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8025 }
8026 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8027 return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8028 }
8029 return 0;
8030}
8031
8032unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8033 if (RetVT.SimpleTy != MVT::v32i16)
8034 return 0;
8035 if ((Subtarget->hasBWI())) {
8036 return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8037 }
8038 return 0;
8039}
8040
8041unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8042 if (RetVT.SimpleTy != MVT::v4i32)
8043 return 0;
8044 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8045 return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8046 }
8047 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8048 return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8049 }
8050 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8051 return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8052 }
8053 return 0;
8054}
8055
8056unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8057 if (RetVT.SimpleTy != MVT::v8i32)
8058 return 0;
8059 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8060 return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8061 }
8062 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8063 return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8064 }
8065 return 0;
8066}
8067
8068unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8069 if (RetVT.SimpleTy != MVT::v16i32)
8070 return 0;
8071 if ((Subtarget->hasAVX512())) {
8072 return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8073 }
8074 return 0;
8075}
8076
8077unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8078 if (RetVT.SimpleTy != MVT::v2i64)
8079 return 0;
8080 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8081 return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8082 }
8083 return 0;
8084}
8085
8086unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8087 if (RetVT.SimpleTy != MVT::v4i64)
8088 return 0;
8089 if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
8090 return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8091 }
8092 return 0;
8093}
8094
8095unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8096 if (RetVT.SimpleTy != MVT::v8i64)
8097 return 0;
8098 if ((Subtarget->hasDQI())) {
8099 return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8100 }
8101 return 0;
8102}
8103
8104unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8105 switch (VT.SimpleTy) {
8106 case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8107 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8108 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8109 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8110 case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8111 case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8112 case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8113 case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8114 case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8115 case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8116 case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8117 case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8118 case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8119 default: return 0;
8120 }
8121}
8122
8123// FastEmit functions for ISD::MULHS.
8124
8125unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8126 if (RetVT.SimpleTy != MVT::v8i16)
8127 return 0;
8128 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8129 return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8130 }
8131 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8132 return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8133 }
8134 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8135 return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8136 }
8137 return 0;
8138}
8139
8140unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8141 if (RetVT.SimpleTy != MVT::v16i16)
8142 return 0;
8143 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8144 return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8145 }
8146 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8147 return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8148 }
8149 return 0;
8150}
8151
8152unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8153 if (RetVT.SimpleTy != MVT::v32i16)
8154 return 0;
8155 if ((Subtarget->hasBWI())) {
8156 return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8157 }
8158 return 0;
8159}
8160
8161unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8162 switch (VT.SimpleTy) {
8163 case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8164 case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8165 case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8166 default: return 0;
8167 }
8168}
8169
8170// FastEmit functions for ISD::MULHU.
8171
8172unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8173 if (RetVT.SimpleTy != MVT::v8i16)
8174 return 0;
8175 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8176 return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8177 }
8178 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8179 return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8180 }
8181 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8182 return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8183 }
8184 return 0;
8185}
8186
8187unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8188 if (RetVT.SimpleTy != MVT::v16i16)
8189 return 0;
8190 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8191 return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8192 }
8193 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8194 return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8195 }
8196 return 0;
8197}
8198
8199unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8200 if (RetVT.SimpleTy != MVT::v32i16)
8201 return 0;
8202 if ((Subtarget->hasBWI())) {
8203 return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8204 }
8205 return 0;
8206}
8207
8208unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8209 switch (VT.SimpleTy) {
8210 case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8211 case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8212 case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8213 default: return 0;
8214 }
8215}
8216
8217// FastEmit functions for ISD::OR.
8218
8219unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8220 if (RetVT.SimpleTy != MVT::i8)
8221 return 0;
8222 return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8223}
8224
8225unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8226 if (RetVT.SimpleTy != MVT::i16)
8227 return 0;
8228 return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8229}
8230
8231unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8232 if (RetVT.SimpleTy != MVT::i32)
8233 return 0;
8234 return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8235}
8236
8237unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8238 if (RetVT.SimpleTy != MVT::i64)
8239 return 0;
8240 return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8241}
8242
8243unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8244 if (RetVT.SimpleTy != MVT::v8i1)
8245 return 0;
8246 if ((Subtarget->hasDQI())) {
8247 return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8248 }
8249 return 0;
8250}
8251
8252unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8253 if (RetVT.SimpleTy != MVT::v16i1)
8254 return 0;
8255 if ((Subtarget->hasAVX512())) {
8256 return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8257 }
8258 return 0;
8259}
8260
8261unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8262 if (RetVT.SimpleTy != MVT::v32i1)
8263 return 0;
8264 if ((Subtarget->hasBWI())) {
8265 return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8266 }
8267 return 0;
8268}
8269
8270unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8271 if (RetVT.SimpleTy != MVT::v64i1)
8272 return 0;
8273 if ((Subtarget->hasBWI())) {
8274 return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8275 }
8276 return 0;
8277}
8278
8279unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8280 if (RetVT.SimpleTy != MVT::v16i8)
8281 return 0;
8282 if ((Subtarget->hasVLX())) {
8283 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8284 }
8285 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8286 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8287 }
8288 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8289 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8290 }
8291 return 0;
8292}
8293
8294unsigned fastEmit_ISD_OR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8295 if (RetVT.SimpleTy != MVT::v32i8)
8296 return 0;
8297 if ((Subtarget->hasVLX())) {
8298 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8299 }
8300 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8301 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8302 }
8303 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8304 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8305 }
8306 return 0;
8307}
8308
8309unsigned fastEmit_ISD_OR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8310 if (RetVT.SimpleTy != MVT::v64i8)
8311 return 0;
8312 if ((Subtarget->hasAVX512())) {
8313 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8314 }
8315 return 0;
8316}
8317
8318unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8319 if (RetVT.SimpleTy != MVT::v8i16)
8320 return 0;
8321 if ((Subtarget->hasVLX())) {
8322 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8323 }
8324 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8325 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8326 }
8327 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8328 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8329 }
8330 return 0;
8331}
8332
8333unsigned fastEmit_ISD_OR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8334 if (RetVT.SimpleTy != MVT::v16i16)
8335 return 0;
8336 if ((Subtarget->hasVLX())) {
8337 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8338 }
8339 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8340 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8341 }
8342 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8343 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8344 }
8345 return 0;
8346}
8347
8348unsigned fastEmit_ISD_OR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8349 if (RetVT.SimpleTy != MVT::v32i16)
8350 return 0;
8351 if ((Subtarget->hasAVX512())) {
8352 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8353 }
8354 return 0;
8355}
8356
8357unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8358 if (RetVT.SimpleTy != MVT::v4i32)
8359 return 0;
8360 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8361 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8362 }
8363 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8364 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8365 }
8366 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8367 return fastEmitInst_rr(X86::VPORDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8368 }
8369 return 0;
8370}
8371
8372unsigned fastEmit_ISD_OR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8373 if (RetVT.SimpleTy != MVT::v8i32)
8374 return 0;
8375 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8376 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8377 }
8378 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8379 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8380 }
8381 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8382 return fastEmitInst_rr(X86::VPORDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8383 }
8384 return 0;
8385}
8386
8387unsigned fastEmit_ISD_OR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8388 if (RetVT.SimpleTy != MVT::v16i32)
8389 return 0;
8390 if ((Subtarget->hasAVX512())) {
8391 return fastEmitInst_rr(X86::VPORDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8392 }
8393 return 0;
8394}
8395
8396unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8397 if (RetVT.SimpleTy != MVT::v2i64)
8398 return 0;
8399 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8400 return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8401 }
8402 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8403 return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8404 }
8405 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8406 return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8407 }
8408 return 0;
8409}
8410
8411unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8412 if (RetVT.SimpleTy != MVT::v4i64)
8413 return 0;
8414 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
8415 return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8416 }
8417 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8418 return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8419 }
8420 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8421 return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8422 }
8423 return 0;
8424}
8425
8426unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8427 if (RetVT.SimpleTy != MVT::v8i64)
8428 return 0;
8429 if ((Subtarget->hasAVX512())) {
8430 return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8431 }
8432 return 0;
8433}
8434
8435unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8436 switch (VT.SimpleTy) {
8437 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8438 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8439 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8440 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8441 case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8442 case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8443 case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8444 case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8445 case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8446 case MVT::v32i8: return fastEmit_ISD_OR_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8447 case MVT::v64i8: return fastEmit_ISD_OR_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8448 case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8449 case MVT::v16i16: return fastEmit_ISD_OR_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8450 case MVT::v32i16: return fastEmit_ISD_OR_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8451 case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8452 case MVT::v8i32: return fastEmit_ISD_OR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8453 case MVT::v16i32: return fastEmit_ISD_OR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8454 case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8455 case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8456 case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8457 default: return 0;
8458 }
8459}
8460
8461// FastEmit functions for ISD::ROTL.
8462
8463unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8464 if (RetVT.SimpleTy != MVT::i8)
8465 return 0;
8466 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
8467 return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
8468}
8469
8470unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8471 if (RetVT.SimpleTy != MVT::v16i8)
8472 return 0;
8473 if ((Subtarget->hasXOP())) {
8474 return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8475 }
8476 return 0;
8477}
8478
8479unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8480 if (RetVT.SimpleTy != MVT::v8i16)
8481 return 0;
8482 if ((Subtarget->hasXOP())) {
8483 return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8484 }
8485 return 0;
8486}
8487
8488unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8489 if (RetVT.SimpleTy != MVT::v4i32)
8490 return 0;
8491 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8492 return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8493 }
8494 if ((Subtarget->hasXOP())) {
8495 return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8496 }
8497 return 0;
8498}
8499
8500unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8501 if (RetVT.SimpleTy != MVT::v8i32)
8502 return 0;
8503 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8504 return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8505 }
8506 return 0;
8507}
8508
8509unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8510 if (RetVT.SimpleTy != MVT::v16i32)
8511 return 0;
8512 if ((Subtarget->hasAVX512())) {
8513 return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8514 }
8515 return 0;
8516}
8517
8518unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8519 if (RetVT.SimpleTy != MVT::v2i64)
8520 return 0;
8521 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8522 return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8523 }
8524 if ((Subtarget->hasXOP())) {
8525 return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8526 }
8527 return 0;
8528}
8529
8530unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8531 if (RetVT.SimpleTy != MVT::v4i64)
8532 return 0;
8533 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8534 return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8535 }
8536 return 0;
8537}
8538
8539unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8540 if (RetVT.SimpleTy != MVT::v8i64)
8541 return 0;
8542 if ((Subtarget->hasAVX512())) {
8543 return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8544 }
8545 return 0;
8546}
8547
8548unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8549 switch (VT.SimpleTy) {
8550 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8551 case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8552 case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8553 case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8554 case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8555 case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8556 case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8557 case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8558 case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8559 default: return 0;
8560 }
8561}
8562
8563// FastEmit functions for ISD::ROTR.
8564
8565unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8566 if (RetVT.SimpleTy != MVT::i8)
8567 return 0;
8568 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
8569 return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
8570}
8571
8572unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8573 if (RetVT.SimpleTy != MVT::v4i32)
8574 return 0;
8575 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8576 return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8577 }
8578 return 0;
8579}
8580
8581unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8582 if (RetVT.SimpleTy != MVT::v8i32)
8583 return 0;
8584 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8585 return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8586 }
8587 return 0;
8588}
8589
8590unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8591 if (RetVT.SimpleTy != MVT::v16i32)
8592 return 0;
8593 if ((Subtarget->hasAVX512())) {
8594 return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8595 }
8596 return 0;
8597}
8598
8599unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8600 if (RetVT.SimpleTy != MVT::v2i64)
8601 return 0;
8602 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8603 return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8604 }
8605 return 0;
8606}
8607
8608unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8609 if (RetVT.SimpleTy != MVT::v4i64)
8610 return 0;
8611 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8612 return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8613 }
8614 return 0;
8615}
8616
8617unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8618 if (RetVT.SimpleTy != MVT::v8i64)
8619 return 0;
8620 if ((Subtarget->hasAVX512())) {
8621 return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8622 }
8623 return 0;
8624}
8625
8626unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8627 switch (VT.SimpleTy) {
8628 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8629 case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8630 case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8631 case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8632 case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8633 case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8634 case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8635 default: return 0;
8636 }
8637}
8638
8639// FastEmit functions for ISD::SADDSAT.
8640
8641unsigned fastEmit_ISD_SADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8642 if (RetVT.SimpleTy != MVT::v16i8)
8643 return 0;
8644 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8645 return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8646 }
8647 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8648 return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8649 }
8650 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8651 return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8652 }
8653 return 0;
8654}
8655
8656unsigned fastEmit_ISD_SADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8657 if (RetVT.SimpleTy != MVT::v32i8)
8658 return 0;
8659 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8660 return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8661 }
8662 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8663 return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8664 }
8665 return 0;
8666}
8667
8668unsigned fastEmit_ISD_SADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8669 if (RetVT.SimpleTy != MVT::v64i8)
8670 return 0;
8671 if ((Subtarget->hasBWI())) {
8672 return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8673 }
8674 return 0;
8675}
8676
8677unsigned fastEmit_ISD_SADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8678 if (RetVT.SimpleTy != MVT::v8i16)
8679 return 0;
8680 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8681 return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8682 }
8683 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8684 return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8685 }
8686 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8687 return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8688 }
8689 return 0;
8690}
8691
8692unsigned fastEmit_ISD_SADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8693 if (RetVT.SimpleTy != MVT::v16i16)
8694 return 0;
8695 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8696 return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8697 }
8698 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8699 return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8700 }
8701 return 0;
8702}
8703
8704unsigned fastEmit_ISD_SADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8705 if (RetVT.SimpleTy != MVT::v32i16)
8706 return 0;
8707 if ((Subtarget->hasBWI())) {
8708 return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8709 }
8710 return 0;
8711}
8712
8713unsigned fastEmit_ISD_SADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8714 switch (VT.SimpleTy) {
8715 case MVT::v16i8: return fastEmit_ISD_SADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8716 case MVT::v32i8: return fastEmit_ISD_SADDSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8717 case MVT::v64i8: return fastEmit_ISD_SADDSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8718 case MVT::v8i16: return fastEmit_ISD_SADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8719 case MVT::v16i16: return fastEmit_ISD_SADDSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8720 case MVT::v32i16: return fastEmit_ISD_SADDSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8721 default: return 0;
8722 }
8723}
8724
8725// FastEmit functions for ISD::SHL.
8726
8727unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8728 if (RetVT.SimpleTy != MVT::i8)
8729 return 0;
8730 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
8731 return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
8732}
8733
8734unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8735 switch (VT.SimpleTy) {
8736 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8737 default: return 0;
8738 }
8739}
8740
8741// FastEmit functions for ISD::SMAX.
8742
8743unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8744 if (RetVT.SimpleTy != MVT::v16i8)
8745 return 0;
8746 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8747 return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8748 }
8749 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8750 return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8751 }
8752 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8753 return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8754 }
8755 return 0;
8756}
8757
8758unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8759 if (RetVT.SimpleTy != MVT::v32i8)
8760 return 0;
8761 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8762 return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8763 }
8764 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8765 return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8766 }
8767 return 0;
8768}
8769
8770unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8771 if (RetVT.SimpleTy != MVT::v64i8)
8772 return 0;
8773 if ((Subtarget->hasBWI())) {
8774 return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8775 }
8776 return 0;
8777}
8778
8779unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8780 if (RetVT.SimpleTy != MVT::v8i16)
8781 return 0;
8782 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8783 return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8784 }
8785 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8786 return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8787 }
8788 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8789 return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8790 }
8791 return 0;
8792}
8793
8794unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8795 if (RetVT.SimpleTy != MVT::v16i16)
8796 return 0;
8797 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8798 return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8799 }
8800 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8801 return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8802 }
8803 return 0;
8804}
8805
8806unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8807 if (RetVT.SimpleTy != MVT::v32i16)
8808 return 0;
8809 if ((Subtarget->hasBWI())) {
8810 return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8811 }
8812 return 0;
8813}
8814
8815unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8816 if (RetVT.SimpleTy != MVT::v4i32)
8817 return 0;
8818 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8819 return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8820 }
8821 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8822 return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8823 }
8824 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8825 return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8826 }
8827 return 0;
8828}
8829
8830unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8831 if (RetVT.SimpleTy != MVT::v8i32)
8832 return 0;
8833 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8834 return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8835 }
8836 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8837 return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8838 }
8839 return 0;
8840}
8841
8842unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8843 if (RetVT.SimpleTy != MVT::v16i32)
8844 return 0;
8845 if ((Subtarget->hasAVX512())) {
8846 return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8847 }
8848 return 0;
8849}
8850
8851unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8852 if (RetVT.SimpleTy != MVT::v2i64)
8853 return 0;
8854 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8855 return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8856 }
8857 return 0;
8858}
8859
8860unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8861 if (RetVT.SimpleTy != MVT::v4i64)
8862 return 0;
8863 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8864 return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8865 }
8866 return 0;
8867}
8868
8869unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8870 if (RetVT.SimpleTy != MVT::v8i64)
8871 return 0;
8872 if ((Subtarget->hasAVX512())) {
8873 return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8874 }
8875 return 0;
8876}
8877
8878unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8879 switch (VT.SimpleTy) {
8880 case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8881 case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8882 case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8883 case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8884 case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8885 case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8886 case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8887 case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8888 case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8889 case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8890 case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8891 case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
8892 default: return 0;
8893 }
8894}
8895
8896// FastEmit functions for ISD::SMIN.
8897
8898unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8899 if (RetVT.SimpleTy != MVT::v16i8)
8900 return 0;
8901 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8902 return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8903 }
8904 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8905 return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8906 }
8907 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8908 return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8909 }
8910 return 0;
8911}
8912
8913unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8914 if (RetVT.SimpleTy != MVT::v32i8)
8915 return 0;
8916 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8917 return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8918 }
8919 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8920 return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8921 }
8922 return 0;
8923}
8924
8925unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8926 if (RetVT.SimpleTy != MVT::v64i8)
8927 return 0;
8928 if ((Subtarget->hasBWI())) {
8929 return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8930 }
8931 return 0;
8932}
8933
8934unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8935 if (RetVT.SimpleTy != MVT::v8i16)
8936 return 0;
8937 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8938 return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8939 }
8940 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
8941 return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8942 }
8943 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8944 return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8945 }
8946 return 0;
8947}
8948
8949unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8950 if (RetVT.SimpleTy != MVT::v16i16)
8951 return 0;
8952 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
8953 return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8954 }
8955 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
8956 return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8957 }
8958 return 0;
8959}
8960
8961unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8962 if (RetVT.SimpleTy != MVT::v32i16)
8963 return 0;
8964 if ((Subtarget->hasBWI())) {
8965 return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8966 }
8967 return 0;
8968}
8969
8970unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8971 if (RetVT.SimpleTy != MVT::v4i32)
8972 return 0;
8973 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8974 return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8975 }
8976 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
8977 return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8978 }
8979 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
8980 return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8981 }
8982 return 0;
8983}
8984
8985unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8986 if (RetVT.SimpleTy != MVT::v8i32)
8987 return 0;
8988 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
8989 return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8990 }
8991 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
8992 return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
8993 }
8994 return 0;
8995}
8996
8997unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
8998 if (RetVT.SimpleTy != MVT::v16i32)
8999 return 0;
9000 if ((Subtarget->hasAVX512())) {
9001 return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9002 }
9003 return 0;
9004}
9005
9006unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9007 if (RetVT.SimpleTy != MVT::v2i64)
9008 return 0;
9009 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9010 return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9011 }
9012 return 0;
9013}
9014
9015unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9016 if (RetVT.SimpleTy != MVT::v4i64)
9017 return 0;
9018 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9019 return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9020 }
9021 return 0;
9022}
9023
9024unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9025 if (RetVT.SimpleTy != MVT::v8i64)
9026 return 0;
9027 if ((Subtarget->hasAVX512())) {
9028 return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9029 }
9030 return 0;
9031}
9032
9033unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9034 switch (VT.SimpleTy) {
9035 case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9036 case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9037 case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9038 case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9039 case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9040 case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9041 case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9042 case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9043 case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9044 case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9045 case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9046 case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9047 default: return 0;
9048 }
9049}
9050
9051// FastEmit functions for ISD::SRA.
9052
9053unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9054 if (RetVT.SimpleTy != MVT::i8)
9055 return 0;
9056 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
9057 return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
9058}
9059
9060unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9061 switch (VT.SimpleTy) {
9062 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9063 default: return 0;
9064 }
9065}
9066
9067// FastEmit functions for ISD::SRL.
9068
9069unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9070 if (RetVT.SimpleTy != MVT::i8)
9071 return 0;
9072 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
9073 return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
9074}
9075
9076unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9077 switch (VT.SimpleTy) {
9078 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9079 default: return 0;
9080 }
9081}
9082
9083// FastEmit functions for ISD::SSUBSAT.
9084
9085unsigned fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9086 if (RetVT.SimpleTy != MVT::v16i8)
9087 return 0;
9088 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9089 return fastEmitInst_rr(X86::VPSUBSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9090 }
9091 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9092 return fastEmitInst_rr(X86::PSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9093 }
9094 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9095 return fastEmitInst_rr(X86::VPSUBSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9096 }
9097 return 0;
9098}
9099
9100unsigned fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9101 if (RetVT.SimpleTy != MVT::v32i8)
9102 return 0;
9103 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9104 return fastEmitInst_rr(X86::VPSUBSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9105 }
9106 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9107 return fastEmitInst_rr(X86::VPSUBSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9108 }
9109 return 0;
9110}
9111
9112unsigned fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9113 if (RetVT.SimpleTy != MVT::v64i8)
9114 return 0;
9115 if ((Subtarget->hasBWI())) {
9116 return fastEmitInst_rr(X86::VPSUBSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9117 }
9118 return 0;
9119}
9120
9121unsigned fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9122 if (RetVT.SimpleTy != MVT::v8i16)
9123 return 0;
9124 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9125 return fastEmitInst_rr(X86::VPSUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9126 }
9127 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9128 return fastEmitInst_rr(X86::PSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9129 }
9130 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9131 return fastEmitInst_rr(X86::VPSUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9132 }
9133 return 0;
9134}
9135
9136unsigned fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9137 if (RetVT.SimpleTy != MVT::v16i16)
9138 return 0;
9139 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9140 return fastEmitInst_rr(X86::VPSUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9141 }
9142 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9143 return fastEmitInst_rr(X86::VPSUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9144 }
9145 return 0;
9146}
9147
9148unsigned fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9149 if (RetVT.SimpleTy != MVT::v32i16)
9150 return 0;
9151 if ((Subtarget->hasBWI())) {
9152 return fastEmitInst_rr(X86::VPSUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9153 }
9154 return 0;
9155}
9156
9157unsigned fastEmit_ISD_SSUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9158 switch (VT.SimpleTy) {
9159 case MVT::v16i8: return fastEmit_ISD_SSUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9160 case MVT::v32i8: return fastEmit_ISD_SSUBSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9161 case MVT::v64i8: return fastEmit_ISD_SSUBSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9162 case MVT::v8i16: return fastEmit_ISD_SSUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9163 case MVT::v16i16: return fastEmit_ISD_SSUBSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9164 case MVT::v32i16: return fastEmit_ISD_SSUBSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9165 default: return 0;
9166 }
9167}
9168
9169// FastEmit functions for ISD::STRICT_FADD.
9170
9171unsigned fastEmit_ISD_STRICT_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9172 if (RetVT.SimpleTy != MVT::f32)
9173 return 0;
9174 if ((Subtarget->hasAVX512())) {
9175 return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9176 }
9177 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9178 return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9179 }
9180 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9181 return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9182 }
9183 if ((!Subtarget->hasSSE1())) {
9184 return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9185 }
9186 return 0;
9187}
9188
9189unsigned fastEmit_ISD_STRICT_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9190 if (RetVT.SimpleTy != MVT::f64)
9191 return 0;
9192 if ((Subtarget->hasAVX512())) {
9193 return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9194 }
9195 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9196 return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9197 }
9198 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9199 return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9200 }
9201 if ((!Subtarget->hasSSE2())) {
9202 return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9203 }
9204 return 0;
9205}
9206
9207unsigned fastEmit_ISD_STRICT_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9208 if (RetVT.SimpleTy != MVT::f80)
9209 return 0;
9210 return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9211}
9212
9213unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9214 if (RetVT.SimpleTy != MVT::v4f32)
9215 return 0;
9216 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9217 return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9218 }
9219 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9220 return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9221 }
9222 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9223 return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9224 }
9225 return 0;
9226}
9227
9228unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9229 if (RetVT.SimpleTy != MVT::v8f32)
9230 return 0;
9231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9232 return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9233 }
9234 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9235 return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9236 }
9237 return 0;
9238}
9239
9240unsigned fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9241 if (RetVT.SimpleTy != MVT::v16f32)
9242 return 0;
9243 if ((Subtarget->hasAVX512())) {
9244 return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9245 }
9246 return 0;
9247}
9248
9249unsigned fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9250 if (RetVT.SimpleTy != MVT::v2f64)
9251 return 0;
9252 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9253 return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9254 }
9255 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9256 return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9257 }
9258 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9259 return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9260 }
9261 return 0;
9262}
9263
9264unsigned fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9265 if (RetVT.SimpleTy != MVT::v4f64)
9266 return 0;
9267 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9268 return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9269 }
9270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9271 return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9272 }
9273 return 0;
9274}
9275
9276unsigned fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9277 if (RetVT.SimpleTy != MVT::v8f64)
9278 return 0;
9279 if ((Subtarget->hasAVX512())) {
9280 return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9281 }
9282 return 0;
9283}
9284
9285unsigned fastEmit_ISD_STRICT_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9286 switch (VT.SimpleTy) {
9287 case MVT::f32: return fastEmit_ISD_STRICT_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9288 case MVT::f64: return fastEmit_ISD_STRICT_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9289 case MVT::f80: return fastEmit_ISD_STRICT_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9290 case MVT::v4f32: return fastEmit_ISD_STRICT_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9291 case MVT::v8f32: return fastEmit_ISD_STRICT_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9292 case MVT::v16f32: return fastEmit_ISD_STRICT_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9293 case MVT::v2f64: return fastEmit_ISD_STRICT_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9294 case MVT::v4f64: return fastEmit_ISD_STRICT_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9295 case MVT::v8f64: return fastEmit_ISD_STRICT_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9296 default: return 0;
9297 }
9298}
9299
9300// FastEmit functions for ISD::STRICT_FDIV.
9301
9302unsigned fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9303 if (RetVT.SimpleTy != MVT::f32)
9304 return 0;
9305 if ((Subtarget->hasAVX512())) {
9306 return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9307 }
9308 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9309 return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9310 }
9311 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9312 return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9313 }
9314 if ((!Subtarget->hasSSE1())) {
9315 return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9316 }
9317 return 0;
9318}
9319
9320unsigned fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9321 if (RetVT.SimpleTy != MVT::f64)
9322 return 0;
9323 if ((Subtarget->hasAVX512())) {
9324 return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9325 }
9326 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9327 return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9328 }
9329 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9330 return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9331 }
9332 if ((!Subtarget->hasSSE2())) {
9333 return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9334 }
9335 return 0;
9336}
9337
9338unsigned fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9339 if (RetVT.SimpleTy != MVT::f80)
9340 return 0;
9341 return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9342}
9343
9344unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9345 if (RetVT.SimpleTy != MVT::v4f32)
9346 return 0;
9347 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9348 return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9349 }
9350 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9351 return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9352 }
9353 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9354 return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9355 }
9356 return 0;
9357}
9358
9359unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9360 if (RetVT.SimpleTy != MVT::v8f32)
9361 return 0;
9362 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9363 return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9364 }
9365 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9366 return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9367 }
9368 return 0;
9369}
9370
9371unsigned fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9372 if (RetVT.SimpleTy != MVT::v16f32)
9373 return 0;
9374 if ((Subtarget->hasAVX512())) {
9375 return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9376 }
9377 return 0;
9378}
9379
9380unsigned fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9381 if (RetVT.SimpleTy != MVT::v2f64)
9382 return 0;
9383 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9384 return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9385 }
9386 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9387 return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9388 }
9389 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9390 return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9391 }
9392 return 0;
9393}
9394
9395unsigned fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9396 if (RetVT.SimpleTy != MVT::v4f64)
9397 return 0;
9398 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9399 return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9400 }
9401 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9402 return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9403 }
9404 return 0;
9405}
9406
9407unsigned fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9408 if (RetVT.SimpleTy != MVT::v8f64)
9409 return 0;
9410 if ((Subtarget->hasAVX512())) {
9411 return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9412 }
9413 return 0;
9414}
9415
9416unsigned fastEmit_ISD_STRICT_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9417 switch (VT.SimpleTy) {
9418 case MVT::f32: return fastEmit_ISD_STRICT_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9419 case MVT::f64: return fastEmit_ISD_STRICT_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9420 case MVT::f80: return fastEmit_ISD_STRICT_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9421 case MVT::v4f32: return fastEmit_ISD_STRICT_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9422 case MVT::v8f32: return fastEmit_ISD_STRICT_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9423 case MVT::v16f32: return fastEmit_ISD_STRICT_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9424 case MVT::v2f64: return fastEmit_ISD_STRICT_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9425 case MVT::v4f64: return fastEmit_ISD_STRICT_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9426 case MVT::v8f64: return fastEmit_ISD_STRICT_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9427 default: return 0;
9428 }
9429}
9430
9431// FastEmit functions for ISD::STRICT_FMUL.
9432
9433unsigned fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9434 if (RetVT.SimpleTy != MVT::f32)
9435 return 0;
9436 if ((Subtarget->hasAVX512())) {
9437 return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9438 }
9439 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9440 return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9441 }
9442 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9443 return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9444 }
9445 if ((!Subtarget->hasSSE1())) {
9446 return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9447 }
9448 return 0;
9449}
9450
9451unsigned fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9452 if (RetVT.SimpleTy != MVT::f64)
9453 return 0;
9454 if ((Subtarget->hasAVX512())) {
9455 return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9456 }
9457 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9458 return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9459 }
9460 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9461 return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9462 }
9463 if ((!Subtarget->hasSSE2())) {
9464 return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9465 }
9466 return 0;
9467}
9468
9469unsigned fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9470 if (RetVT.SimpleTy != MVT::f80)
9471 return 0;
9472 return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9473}
9474
9475unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9476 if (RetVT.SimpleTy != MVT::v4f32)
9477 return 0;
9478 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9479 return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9480 }
9481 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9482 return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9483 }
9484 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9485 return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9486 }
9487 return 0;
9488}
9489
9490unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9491 if (RetVT.SimpleTy != MVT::v8f32)
9492 return 0;
9493 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9494 return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9495 }
9496 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9497 return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9498 }
9499 return 0;
9500}
9501
9502unsigned fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9503 if (RetVT.SimpleTy != MVT::v16f32)
9504 return 0;
9505 if ((Subtarget->hasAVX512())) {
9506 return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9507 }
9508 return 0;
9509}
9510
9511unsigned fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9512 if (RetVT.SimpleTy != MVT::v2f64)
9513 return 0;
9514 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9515 return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9516 }
9517 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9518 return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9519 }
9520 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9521 return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9522 }
9523 return 0;
9524}
9525
9526unsigned fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9527 if (RetVT.SimpleTy != MVT::v4f64)
9528 return 0;
9529 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9530 return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9531 }
9532 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9533 return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9534 }
9535 return 0;
9536}
9537
9538unsigned fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9539 if (RetVT.SimpleTy != MVT::v8f64)
9540 return 0;
9541 if ((Subtarget->hasAVX512())) {
9542 return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9543 }
9544 return 0;
9545}
9546
9547unsigned fastEmit_ISD_STRICT_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9548 switch (VT.SimpleTy) {
9549 case MVT::f32: return fastEmit_ISD_STRICT_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9550 case MVT::f64: return fastEmit_ISD_STRICT_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9551 case MVT::f80: return fastEmit_ISD_STRICT_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9552 case MVT::v4f32: return fastEmit_ISD_STRICT_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9553 case MVT::v8f32: return fastEmit_ISD_STRICT_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9554 case MVT::v16f32: return fastEmit_ISD_STRICT_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9555 case MVT::v2f64: return fastEmit_ISD_STRICT_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9556 case MVT::v4f64: return fastEmit_ISD_STRICT_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9557 case MVT::v8f64: return fastEmit_ISD_STRICT_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9558 default: return 0;
9559 }
9560}
9561
9562// FastEmit functions for ISD::STRICT_FSUB.
9563
9564unsigned fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9565 if (RetVT.SimpleTy != MVT::f32)
9566 return 0;
9567 if ((Subtarget->hasAVX512())) {
9568 return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9569 }
9570 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9571 return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9572 }
9573 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9574 return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9575 }
9576 if ((!Subtarget->hasSSE1())) {
9577 return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9578 }
9579 return 0;
9580}
9581
9582unsigned fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9583 if (RetVT.SimpleTy != MVT::f64)
9584 return 0;
9585 if ((Subtarget->hasAVX512())) {
9586 return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9587 }
9588 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9589 return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9590 }
9591 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
9592 return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9593 }
9594 if ((!Subtarget->hasSSE2())) {
9595 return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9596 }
9597 return 0;
9598}
9599
9600unsigned fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9601 if (RetVT.SimpleTy != MVT::f80)
9602 return 0;
9603 return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9604}
9605
9606unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9607 if (RetVT.SimpleTy != MVT::v4f32)
9608 return 0;
9609 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9610 return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9611 }
9612 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
9613 return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9614 }
9615 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9616 return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9617 }
9618 return 0;
9619}
9620
9621unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9622 if (RetVT.SimpleTy != MVT::v8f32)
9623 return 0;
9624 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9625 return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9626 }
9627 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9628 return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9629 }
9630 return 0;
9631}
9632
9633unsigned fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9634 if (RetVT.SimpleTy != MVT::v16f32)
9635 return 0;
9636 if ((Subtarget->hasAVX512())) {
9637 return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9638 }
9639 return 0;
9640}
9641
9642unsigned fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9643 if (RetVT.SimpleTy != MVT::v2f64)
9644 return 0;
9645 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9646 return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9647 }
9648 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9649 return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9650 }
9651 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9652 return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9653 }
9654 return 0;
9655}
9656
9657unsigned fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9658 if (RetVT.SimpleTy != MVT::v4f64)
9659 return 0;
9660 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9661 return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9662 }
9663 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9664 return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9665 }
9666 return 0;
9667}
9668
9669unsigned fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9670 if (RetVT.SimpleTy != MVT::v8f64)
9671 return 0;
9672 if ((Subtarget->hasAVX512())) {
9673 return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9674 }
9675 return 0;
9676}
9677
9678unsigned fastEmit_ISD_STRICT_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9679 switch (VT.SimpleTy) {
9680 case MVT::f32: return fastEmit_ISD_STRICT_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9681 case MVT::f64: return fastEmit_ISD_STRICT_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9682 case MVT::f80: return fastEmit_ISD_STRICT_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9683 case MVT::v4f32: return fastEmit_ISD_STRICT_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9684 case MVT::v8f32: return fastEmit_ISD_STRICT_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9685 case MVT::v16f32: return fastEmit_ISD_STRICT_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9686 case MVT::v2f64: return fastEmit_ISD_STRICT_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9687 case MVT::v4f64: return fastEmit_ISD_STRICT_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9688 case MVT::v8f64: return fastEmit_ISD_STRICT_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9689 default: return 0;
9690 }
9691}
9692
9693// FastEmit functions for ISD::SUB.
9694
9695unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9696 if (RetVT.SimpleTy != MVT::i8)
9697 return 0;
9698 return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9699}
9700
9701unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9702 if (RetVT.SimpleTy != MVT::i16)
9703 return 0;
9704 return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9705}
9706
9707unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9708 if (RetVT.SimpleTy != MVT::i32)
9709 return 0;
9710 return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9711}
9712
9713unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9714 if (RetVT.SimpleTy != MVT::i64)
9715 return 0;
9716 return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9717}
9718
9719unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9720 if (RetVT.SimpleTy != MVT::v16i8)
9721 return 0;
9722 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9723 return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9724 }
9725 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9726 return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9727 }
9728 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9729 return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9730 }
9731 return 0;
9732}
9733
9734unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9735 if (RetVT.SimpleTy != MVT::v32i8)
9736 return 0;
9737 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9738 return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9739 }
9740 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9741 return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9742 }
9743 return 0;
9744}
9745
9746unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9747 if (RetVT.SimpleTy != MVT::v64i8)
9748 return 0;
9749 if ((Subtarget->hasBWI())) {
9750 return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9751 }
9752 return 0;
9753}
9754
9755unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9756 if (RetVT.SimpleTy != MVT::v8i16)
9757 return 0;
9758 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9759 return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9760 }
9761 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9762 return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9763 }
9764 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9765 return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9766 }
9767 return 0;
9768}
9769
9770unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9771 if (RetVT.SimpleTy != MVT::v16i16)
9772 return 0;
9773 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9774 return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9775 }
9776 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9777 return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9778 }
9779 return 0;
9780}
9781
9782unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9783 if (RetVT.SimpleTy != MVT::v32i16)
9784 return 0;
9785 if ((Subtarget->hasBWI())) {
9786 return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9787 }
9788 return 0;
9789}
9790
9791unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9792 if (RetVT.SimpleTy != MVT::v4i32)
9793 return 0;
9794 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9795 return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9796 }
9797 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9798 return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9799 }
9800 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9801 return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9802 }
9803 return 0;
9804}
9805
9806unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9807 if (RetVT.SimpleTy != MVT::v8i32)
9808 return 0;
9809 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9810 return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9811 }
9812 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9813 return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9814 }
9815 return 0;
9816}
9817
9818unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9819 if (RetVT.SimpleTy != MVT::v16i32)
9820 return 0;
9821 if ((Subtarget->hasAVX512())) {
9822 return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9823 }
9824 return 0;
9825}
9826
9827unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9828 if (RetVT.SimpleTy != MVT::v2i64)
9829 return 0;
9830 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9831 return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9832 }
9833 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9834 return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9835 }
9836 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
9837 return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9838 }
9839 return 0;
9840}
9841
9842unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9843 if (RetVT.SimpleTy != MVT::v4i64)
9844 return 0;
9845 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
9846 return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9847 }
9848 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
9849 return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9850 }
9851 return 0;
9852}
9853
9854unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9855 if (RetVT.SimpleTy != MVT::v8i64)
9856 return 0;
9857 if ((Subtarget->hasAVX512())) {
9858 return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9859 }
9860 return 0;
9861}
9862
9863unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9864 switch (VT.SimpleTy) {
9865 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9866 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9867 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9868 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9869 case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9870 case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9871 case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9872 case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9873 case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9874 case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9875 case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9876 case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9877 case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9878 case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9879 case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9880 case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9881 default: return 0;
9882 }
9883}
9884
9885// FastEmit functions for ISD::UADDSAT.
9886
9887unsigned fastEmit_ISD_UADDSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9888 if (RetVT.SimpleTy != MVT::v16i8)
9889 return 0;
9890 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9891 return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9892 }
9893 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9894 return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9895 }
9896 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9897 return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9898 }
9899 return 0;
9900}
9901
9902unsigned fastEmit_ISD_UADDSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9903 if (RetVT.SimpleTy != MVT::v32i8)
9904 return 0;
9905 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9906 return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9907 }
9908 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9909 return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9910 }
9911 return 0;
9912}
9913
9914unsigned fastEmit_ISD_UADDSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9915 if (RetVT.SimpleTy != MVT::v64i8)
9916 return 0;
9917 if ((Subtarget->hasBWI())) {
9918 return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9919 }
9920 return 0;
9921}
9922
9923unsigned fastEmit_ISD_UADDSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9924 if (RetVT.SimpleTy != MVT::v8i16)
9925 return 0;
9926 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9927 return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9928 }
9929 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9930 return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9931 }
9932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9933 return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9934 }
9935 return 0;
9936}
9937
9938unsigned fastEmit_ISD_UADDSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9939 if (RetVT.SimpleTy != MVT::v16i16)
9940 return 0;
9941 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9942 return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9943 }
9944 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9945 return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9946 }
9947 return 0;
9948}
9949
9950unsigned fastEmit_ISD_UADDSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9951 if (RetVT.SimpleTy != MVT::v32i16)
9952 return 0;
9953 if ((Subtarget->hasBWI())) {
9954 return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9955 }
9956 return 0;
9957}
9958
9959unsigned fastEmit_ISD_UADDSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9960 switch (VT.SimpleTy) {
9961 case MVT::v16i8: return fastEmit_ISD_UADDSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9962 case MVT::v32i8: return fastEmit_ISD_UADDSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9963 case MVT::v64i8: return fastEmit_ISD_UADDSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9964 case MVT::v8i16: return fastEmit_ISD_UADDSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9965 case MVT::v16i16: return fastEmit_ISD_UADDSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9966 case MVT::v32i16: return fastEmit_ISD_UADDSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
9967 default: return 0;
9968 }
9969}
9970
9971// FastEmit functions for ISD::UMAX.
9972
9973unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9974 if (RetVT.SimpleTy != MVT::v16i8)
9975 return 0;
9976 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9977 return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9978 }
9979 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
9980 return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9981 }
9982 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9983 return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9984 }
9985 return 0;
9986}
9987
9988unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
9989 if (RetVT.SimpleTy != MVT::v32i8)
9990 return 0;
9991 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
9992 return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9993 }
9994 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
9995 return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
9996 }
9997 return 0;
9998}
9999
10000unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10001 if (RetVT.SimpleTy != MVT::v64i8)
10002 return 0;
10003 if ((Subtarget->hasBWI())) {
10004 return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10005 }
10006 return 0;
10007}
10008
10009unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10010 if (RetVT.SimpleTy != MVT::v8i16)
10011 return 0;
10012 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10013 return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10014 }
10015 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10016 return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10017 }
10018 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10019 return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10020 }
10021 return 0;
10022}
10023
10024unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10025 if (RetVT.SimpleTy != MVT::v16i16)
10026 return 0;
10027 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10028 return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10029 }
10030 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10031 return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10032 }
10033 return 0;
10034}
10035
10036unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10037 if (RetVT.SimpleTy != MVT::v32i16)
10038 return 0;
10039 if ((Subtarget->hasBWI())) {
10040 return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10041 }
10042 return 0;
10043}
10044
10045unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10046 if (RetVT.SimpleTy != MVT::v4i32)
10047 return 0;
10048 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10049 return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10050 }
10051 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10052 return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10053 }
10054 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10055 return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10056 }
10057 return 0;
10058}
10059
10060unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10061 if (RetVT.SimpleTy != MVT::v8i32)
10062 return 0;
10063 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10064 return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10065 }
10066 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10067 return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10068 }
10069 return 0;
10070}
10071
10072unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10073 if (RetVT.SimpleTy != MVT::v16i32)
10074 return 0;
10075 if ((Subtarget->hasAVX512())) {
10076 return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10077 }
10078 return 0;
10079}
10080
10081unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10082 if (RetVT.SimpleTy != MVT::v2i64)
10083 return 0;
10084 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10085 return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10086 }
10087 return 0;
10088}
10089
10090unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10091 if (RetVT.SimpleTy != MVT::v4i64)
10092 return 0;
10093 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10094 return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10095 }
10096 return 0;
10097}
10098
10099unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10100 if (RetVT.SimpleTy != MVT::v8i64)
10101 return 0;
10102 if ((Subtarget->hasAVX512())) {
10103 return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10104 }
10105 return 0;
10106}
10107
10108unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10109 switch (VT.SimpleTy) {
10110 case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10111 case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10112 case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10113 case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10114 case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10115 case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10116 case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10117 case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10118 case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10119 case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10120 case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10121 case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10122 default: return 0;
10123 }
10124}
10125
10126// FastEmit functions for ISD::UMIN.
10127
10128unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10129 if (RetVT.SimpleTy != MVT::v16i8)
10130 return 0;
10131 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10132 return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10133 }
10134 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10135 return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10136 }
10137 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10138 return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10139 }
10140 return 0;
10141}
10142
10143unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10144 if (RetVT.SimpleTy != MVT::v32i8)
10145 return 0;
10146 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10147 return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10148 }
10149 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10150 return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10151 }
10152 return 0;
10153}
10154
10155unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10156 if (RetVT.SimpleTy != MVT::v64i8)
10157 return 0;
10158 if ((Subtarget->hasBWI())) {
10159 return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10160 }
10161 return 0;
10162}
10163
10164unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10165 if (RetVT.SimpleTy != MVT::v8i16)
10166 return 0;
10167 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10168 return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10169 }
10170 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10171 return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10172 }
10173 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10174 return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10175 }
10176 return 0;
10177}
10178
10179unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10180 if (RetVT.SimpleTy != MVT::v16i16)
10181 return 0;
10182 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10183 return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10184 }
10185 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10186 return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10187 }
10188 return 0;
10189}
10190
10191unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10192 if (RetVT.SimpleTy != MVT::v32i16)
10193 return 0;
10194 if ((Subtarget->hasBWI())) {
10195 return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10196 }
10197 return 0;
10198}
10199
10200unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10201 if (RetVT.SimpleTy != MVT::v4i32)
10202 return 0;
10203 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10204 return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10205 }
10206 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
10207 return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10208 }
10209 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10210 return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10211 }
10212 return 0;
10213}
10214
10215unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10216 if (RetVT.SimpleTy != MVT::v8i32)
10217 return 0;
10218 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10219 return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10220 }
10221 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10222 return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10223 }
10224 return 0;
10225}
10226
10227unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10228 if (RetVT.SimpleTy != MVT::v16i32)
10229 return 0;
10230 if ((Subtarget->hasAVX512())) {
10231 return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10232 }
10233 return 0;
10234}
10235
10236unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10237 if (RetVT.SimpleTy != MVT::v2i64)
10238 return 0;
10239 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10240 return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10241 }
10242 return 0;
10243}
10244
10245unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10246 if (RetVT.SimpleTy != MVT::v4i64)
10247 return 0;
10248 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10249 return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10250 }
10251 return 0;
10252}
10253
10254unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10255 if (RetVT.SimpleTy != MVT::v8i64)
10256 return 0;
10257 if ((Subtarget->hasAVX512())) {
10258 return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10259 }
10260 return 0;
10261}
10262
10263unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10264 switch (VT.SimpleTy) {
10265 case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10266 case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10267 case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10268 case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10269 case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10270 case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10271 case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10272 case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10273 case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10274 case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10275 case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10276 case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10277 default: return 0;
10278 }
10279}
10280
10281// FastEmit functions for ISD::USUBSAT.
10282
10283unsigned fastEmit_ISD_USUBSAT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10284 if (RetVT.SimpleTy != MVT::v16i8)
10285 return 0;
10286 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10287 return fastEmitInst_rr(X86::VPSUBUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10288 }
10289 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10290 return fastEmitInst_rr(X86::PSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10291 }
10292 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10293 return fastEmitInst_rr(X86::VPSUBUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10294 }
10295 return 0;
10296}
10297
10298unsigned fastEmit_ISD_USUBSAT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10299 if (RetVT.SimpleTy != MVT::v32i8)
10300 return 0;
10301 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10302 return fastEmitInst_rr(X86::VPSUBUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10303 }
10304 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10305 return fastEmitInst_rr(X86::VPSUBUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10306 }
10307 return 0;
10308}
10309
10310unsigned fastEmit_ISD_USUBSAT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10311 if (RetVT.SimpleTy != MVT::v64i8)
10312 return 0;
10313 if ((Subtarget->hasBWI())) {
10314 return fastEmitInst_rr(X86::VPSUBUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10315 }
10316 return 0;
10317}
10318
10319unsigned fastEmit_ISD_USUBSAT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10320 if (RetVT.SimpleTy != MVT::v8i16)
10321 return 0;
10322 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10323 return fastEmitInst_rr(X86::VPSUBUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10324 }
10325 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10326 return fastEmitInst_rr(X86::PSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10327 }
10328 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10329 return fastEmitInst_rr(X86::VPSUBUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10330 }
10331 return 0;
10332}
10333
10334unsigned fastEmit_ISD_USUBSAT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10335 if (RetVT.SimpleTy != MVT::v16i16)
10336 return 0;
10337 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10338 return fastEmitInst_rr(X86::VPSUBUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10339 }
10340 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10341 return fastEmitInst_rr(X86::VPSUBUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10342 }
10343 return 0;
10344}
10345
10346unsigned fastEmit_ISD_USUBSAT_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10347 if (RetVT.SimpleTy != MVT::v32i16)
10348 return 0;
10349 if ((Subtarget->hasBWI())) {
10350 return fastEmitInst_rr(X86::VPSUBUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10351 }
10352 return 0;
10353}
10354
10355unsigned fastEmit_ISD_USUBSAT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10356 switch (VT.SimpleTy) {
10357 case MVT::v16i8: return fastEmit_ISD_USUBSAT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10358 case MVT::v32i8: return fastEmit_ISD_USUBSAT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10359 case MVT::v64i8: return fastEmit_ISD_USUBSAT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10360 case MVT::v8i16: return fastEmit_ISD_USUBSAT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10361 case MVT::v16i16: return fastEmit_ISD_USUBSAT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10362 case MVT::v32i16: return fastEmit_ISD_USUBSAT_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10363 default: return 0;
10364 }
10365}
10366
10367// FastEmit functions for ISD::XOR.
10368
10369unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10370 if (RetVT.SimpleTy != MVT::i8)
10371 return 0;
10372 return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10373}
10374
10375unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10376 if (RetVT.SimpleTy != MVT::i16)
10377 return 0;
10378 return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10379}
10380
10381unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10382 if (RetVT.SimpleTy != MVT::i32)
10383 return 0;
10384 return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10385}
10386
10387unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10388 if (RetVT.SimpleTy != MVT::i64)
10389 return 0;
10390 return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10391}
10392
10393unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10394 if (RetVT.SimpleTy != MVT::v8i1)
10395 return 0;
10396 if ((Subtarget->hasDQI())) {
10397 return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10398 }
10399 return 0;
10400}
10401
10402unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10403 if (RetVT.SimpleTy != MVT::v16i1)
10404 return 0;
10405 if ((Subtarget->hasAVX512())) {
10406 return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10407 }
10408 return 0;
10409}
10410
10411unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10412 if (RetVT.SimpleTy != MVT::v32i1)
10413 return 0;
10414 if ((Subtarget->hasBWI())) {
10415 return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10416 }
10417 return 0;
10418}
10419
10420unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10421 if (RetVT.SimpleTy != MVT::v64i1)
10422 return 0;
10423 if ((Subtarget->hasBWI())) {
10424 return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10425 }
10426 return 0;
10427}
10428
10429unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10430 if (RetVT.SimpleTy != MVT::v16i8)
10431 return 0;
10432 if ((Subtarget->hasVLX())) {
10433 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10434 }
10435 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10436 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10437 }
10438 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10439 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10440 }
10441 return 0;
10442}
10443
10444unsigned fastEmit_ISD_XOR_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10445 if (RetVT.SimpleTy != MVT::v32i8)
10446 return 0;
10447 if ((Subtarget->hasVLX())) {
10448 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10449 }
10450 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10451 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10452 }
10453 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10454 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10455 }
10456 return 0;
10457}
10458
10459unsigned fastEmit_ISD_XOR_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10460 if (RetVT.SimpleTy != MVT::v64i8)
10461 return 0;
10462 if ((Subtarget->hasAVX512())) {
10463 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10464 }
10465 return 0;
10466}
10467
10468unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10469 if (RetVT.SimpleTy != MVT::v8i16)
10470 return 0;
10471 if ((Subtarget->hasVLX())) {
10472 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10473 }
10474 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10475 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10476 }
10477 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10478 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10479 }
10480 return 0;
10481}
10482
10483unsigned fastEmit_ISD_XOR_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10484 if (RetVT.SimpleTy != MVT::v16i16)
10485 return 0;
10486 if ((Subtarget->hasVLX())) {
10487 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10488 }
10489 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10490 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10491 }
10492 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10493 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10494 }
10495 return 0;
10496}
10497
10498unsigned fastEmit_ISD_XOR_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10499 if (RetVT.SimpleTy != MVT::v32i16)
10500 return 0;
10501 if ((Subtarget->hasAVX512())) {
10502 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10503 }
10504 return 0;
10505}
10506
10507unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10508 if (RetVT.SimpleTy != MVT::v4i32)
10509 return 0;
10510 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10511 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10512 }
10513 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10514 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10515 }
10516 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10517 return fastEmitInst_rr(X86::VPXORDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10518 }
10519 return 0;
10520}
10521
10522unsigned fastEmit_ISD_XOR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10523 if (RetVT.SimpleTy != MVT::v8i32)
10524 return 0;
10525 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10526 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10527 }
10528 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10529 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10530 }
10531 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10532 return fastEmitInst_rr(X86::VPXORDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10533 }
10534 return 0;
10535}
10536
10537unsigned fastEmit_ISD_XOR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10538 if (RetVT.SimpleTy != MVT::v16i32)
10539 return 0;
10540 if ((Subtarget->hasAVX512())) {
10541 return fastEmitInst_rr(X86::VPXORDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10542 }
10543 return 0;
10544}
10545
10546unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10547 if (RetVT.SimpleTy != MVT::v2i64)
10548 return 0;
10549 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10550 return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10551 }
10552 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10553 return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10554 }
10555 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10556 return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10557 }
10558 return 0;
10559}
10560
10561unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10562 if (RetVT.SimpleTy != MVT::v4i64)
10563 return 0;
10564 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10565 return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10566 }
10567 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10568 return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10569 }
10570 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10571 return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10572 }
10573 return 0;
10574}
10575
10576unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10577 if (RetVT.SimpleTy != MVT::v8i64)
10578 return 0;
10579 if ((Subtarget->hasAVX512())) {
10580 return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10581 }
10582 return 0;
10583}
10584
10585unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10586 switch (VT.SimpleTy) {
10587 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10588 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10589 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10590 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10591 case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10592 case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10593 case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10594 case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10595 case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10596 case MVT::v32i8: return fastEmit_ISD_XOR_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10597 case MVT::v64i8: return fastEmit_ISD_XOR_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10598 case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10599 case MVT::v16i16: return fastEmit_ISD_XOR_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10600 case MVT::v32i16: return fastEmit_ISD_XOR_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10601 case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10602 case MVT::v8i32: return fastEmit_ISD_XOR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10603 case MVT::v16i32: return fastEmit_ISD_XOR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10604 case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10605 case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10606 case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10607 default: return 0;
10608 }
10609}
10610
10611// FastEmit functions for X86ISD::ADDSUB.
10612
10613unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10614 if (RetVT.SimpleTy != MVT::v4f32)
10615 return 0;
10616 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
10617 return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10618 }
10619 if ((Subtarget->hasAVX())) {
10620 return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10621 }
10622 return 0;
10623}
10624
10625unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10626 if (RetVT.SimpleTy != MVT::v8f32)
10627 return 0;
10628 if ((Subtarget->hasAVX())) {
10629 return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10630 }
10631 return 0;
10632}
10633
10634unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10635 if (RetVT.SimpleTy != MVT::v2f64)
10636 return 0;
10637 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
10638 return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10639 }
10640 if ((Subtarget->hasAVX())) {
10641 return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10642 }
10643 return 0;
10644}
10645
10646unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10647 if (RetVT.SimpleTy != MVT::v4f64)
10648 return 0;
10649 if ((Subtarget->hasAVX())) {
10650 return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10651 }
10652 return 0;
10653}
10654
10655unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10656 switch (VT.SimpleTy) {
10657 case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10658 case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10659 case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10660 case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10661 default: return 0;
10662 }
10663}
10664
10665// FastEmit functions for X86ISD::ANDNP.
10666
10667unsigned fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10668 if (RetVT.SimpleTy != MVT::v16i8)
10669 return 0;
10670 if ((Subtarget->hasVLX())) {
10671 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10672 }
10673 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10674 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10675 }
10676 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10677 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10678 }
10679 return 0;
10680}
10681
10682unsigned fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10683 if (RetVT.SimpleTy != MVT::v32i8)
10684 return 0;
10685 if ((Subtarget->hasVLX())) {
10686 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10687 }
10688 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10689 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10690 }
10691 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10692 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10693 }
10694 return 0;
10695}
10696
10697unsigned fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10698 if (RetVT.SimpleTy != MVT::v64i8)
10699 return 0;
10700 if ((Subtarget->hasAVX512())) {
10701 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10702 }
10703 return 0;
10704}
10705
10706unsigned fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10707 if (RetVT.SimpleTy != MVT::v8i16)
10708 return 0;
10709 if ((Subtarget->hasVLX())) {
10710 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10711 }
10712 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10713 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10714 }
10715 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10716 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10717 }
10718 return 0;
10719}
10720
10721unsigned fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10722 if (RetVT.SimpleTy != MVT::v16i16)
10723 return 0;
10724 if ((Subtarget->hasVLX())) {
10725 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10726 }
10727 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10728 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10729 }
10730 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10731 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10732 }
10733 return 0;
10734}
10735
10736unsigned fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10737 if (RetVT.SimpleTy != MVT::v32i16)
10738 return 0;
10739 if ((Subtarget->hasAVX512())) {
10740 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10741 }
10742 return 0;
10743}
10744
10745unsigned fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10746 if (RetVT.SimpleTy != MVT::v4i32)
10747 return 0;
10748 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10749 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10750 }
10751 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10752 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10753 }
10754 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10755 return fastEmitInst_rr(X86::VPANDNDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10756 }
10757 return 0;
10758}
10759
10760unsigned fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10761 if (RetVT.SimpleTy != MVT::v8i32)
10762 return 0;
10763 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10764 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10765 }
10766 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10767 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10768 }
10769 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10770 return fastEmitInst_rr(X86::VPANDNDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10771 }
10772 return 0;
10773}
10774
10775unsigned fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10776 if (RetVT.SimpleTy != MVT::v16i32)
10777 return 0;
10778 if ((Subtarget->hasAVX512())) {
10779 return fastEmitInst_rr(X86::VPANDNDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10780 }
10781 return 0;
10782}
10783
10784unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10785 if (RetVT.SimpleTy != MVT::v2i64)
10786 return 0;
10787 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10788 return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10789 }
10790 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10791 return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10792 }
10793 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
10794 return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10795 }
10796 return 0;
10797}
10798
10799unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10800 if (RetVT.SimpleTy != MVT::v4i64)
10801 return 0;
10802 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
10803 return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10804 }
10805 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
10806 return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10807 }
10808 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
10809 return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10810 }
10811 return 0;
10812}
10813
10814unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10815 if (RetVT.SimpleTy != MVT::v8i64)
10816 return 0;
10817 if ((Subtarget->hasAVX512())) {
10818 return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10819 }
10820 return 0;
10821}
10822
10823unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10824 switch (VT.SimpleTy) {
10825 case MVT::v16i8: return fastEmit_X86ISD_ANDNP_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10826 case MVT::v32i8: return fastEmit_X86ISD_ANDNP_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10827 case MVT::v64i8: return fastEmit_X86ISD_ANDNP_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10828 case MVT::v8i16: return fastEmit_X86ISD_ANDNP_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10829 case MVT::v16i16: return fastEmit_X86ISD_ANDNP_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10830 case MVT::v32i16: return fastEmit_X86ISD_ANDNP_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10831 case MVT::v4i32: return fastEmit_X86ISD_ANDNP_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10832 case MVT::v8i32: return fastEmit_X86ISD_ANDNP_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10833 case MVT::v16i32: return fastEmit_X86ISD_ANDNP_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10834 case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10835 case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10836 case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10837 default: return 0;
10838 }
10839}
10840
10841// FastEmit functions for X86ISD::AVG.
10842
10843unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10844 if (RetVT.SimpleTy != MVT::v16i8)
10845 return 0;
10846 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10847 return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10848 }
10849 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10850 return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10851 }
10852 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10853 return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10854 }
10855 return 0;
10856}
10857
10858unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10859 if (RetVT.SimpleTy != MVT::v32i8)
10860 return 0;
10861 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10862 return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10863 }
10864 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10865 return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10866 }
10867 return 0;
10868}
10869
10870unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10871 if (RetVT.SimpleTy != MVT::v64i8)
10872 return 0;
10873 if ((Subtarget->hasBWI())) {
10874 return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10875 }
10876 return 0;
10877}
10878
10879unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10880 if (RetVT.SimpleTy != MVT::v8i16)
10881 return 0;
10882 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10883 return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10884 }
10885 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
10886 return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10887 }
10888 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10889 return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10890 }
10891 return 0;
10892}
10893
10894unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10895 if (RetVT.SimpleTy != MVT::v16i16)
10896 return 0;
10897 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
10898 return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10899 }
10900 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
10901 return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10902 }
10903 return 0;
10904}
10905
10906unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10907 if (RetVT.SimpleTy != MVT::v32i16)
10908 return 0;
10909 if ((Subtarget->hasBWI())) {
10910 return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10911 }
10912 return 0;
10913}
10914
10915unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10916 switch (VT.SimpleTy) {
10917 case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10918 case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10919 case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10920 case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10921 case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10922 case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10923 default: return 0;
10924 }
10925}
10926
10927// FastEmit functions for X86ISD::BEXTR.
10928
10929unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10930 if (RetVT.SimpleTy != MVT::i32)
10931 return 0;
10932 if ((Subtarget->hasBMI())) {
10933 return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10934 }
10935 return 0;
10936}
10937
10938unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10939 if (RetVT.SimpleTy != MVT::i64)
10940 return 0;
10941 if ((Subtarget->hasBMI())) {
10942 return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10943 }
10944 return 0;
10945}
10946
10947unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10948 switch (VT.SimpleTy) {
10949 case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10950 case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10951 default: return 0;
10952 }
10953}
10954
10955// FastEmit functions for X86ISD::BT.
10956
10957unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10958 if (RetVT.SimpleTy != MVT::i32)
10959 return 0;
10960 return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10961}
10962
10963unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10964 if (RetVT.SimpleTy != MVT::i32)
10965 return 0;
10966 return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10967}
10968
10969unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10970 if (RetVT.SimpleTy != MVT::i32)
10971 return 0;
10972 return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10973}
10974
10975unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10976 switch (VT.SimpleTy) {
10977 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10978 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10979 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
10980 default: return 0;
10981 }
10982}
10983
10984// FastEmit functions for X86ISD::BZHI.
10985
10986unsigned fastEmit_X86ISD_BZHI_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10987 if (RetVT.SimpleTy != MVT::i32)
10988 return 0;
10989 if ((Subtarget->hasBMI2())) {
10990 return fastEmitInst_rr(X86::BZHI32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
10991 }
10992 return 0;
10993}
10994
10995unsigned fastEmit_X86ISD_BZHI_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
10996 if (RetVT.SimpleTy != MVT::i64)
10997 return 0;
10998 if ((Subtarget->hasBMI2())) {
10999 return fastEmitInst_rr(X86::BZHI64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11000 }
11001 return 0;
11002}
11003
11004unsigned fastEmit_X86ISD_BZHI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11005 switch (VT.SimpleTy) {
11006 case MVT::i32: return fastEmit_X86ISD_BZHI_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11007 case MVT::i64: return fastEmit_X86ISD_BZHI_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11008 default: return 0;
11009 }
11010}
11011
11012// FastEmit functions for X86ISD::CMP.
11013
11014unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11015 if (RetVT.SimpleTy != MVT::i32)
11016 return 0;
11017 return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11018}
11019
11020unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11021 if (RetVT.SimpleTy != MVT::i32)
11022 return 0;
11023 return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11024}
11025
11026unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11027 if (RetVT.SimpleTy != MVT::i32)
11028 return 0;
11029 return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11030}
11031
11032unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11033 if (RetVT.SimpleTy != MVT::i32)
11034 return 0;
11035 return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11036}
11037
11038unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11039 switch (VT.SimpleTy) {
11040 case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11041 case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11042 case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11043 case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11044 default: return 0;
11045 }
11046}
11047
11048// FastEmit functions for X86ISD::COMI.
11049
11050unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11051 if (RetVT.SimpleTy != MVT::i32)
11052 return 0;
11053 if ((Subtarget->hasAVX512())) {
11054 return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11055 }
11056 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11057 return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11058 }
11059 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11060 return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11061 }
11062 return 0;
11063}
11064
11065unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11066 if (RetVT.SimpleTy != MVT::i32)
11067 return 0;
11068 if ((Subtarget->hasAVX512())) {
11069 return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11070 }
11071 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11072 return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11073 }
11074 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11075 return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11076 }
11077 return 0;
11078}
11079
11080unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11081 switch (VT.SimpleTy) {
11082 case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11083 case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11084 default: return 0;
11085 }
11086}
11087
11088// FastEmit functions for X86ISD::CVTNE2PS2BF16.
11089
11090unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11091 if (RetVT.SimpleTy != MVT::v8i16)
11092 return 0;
11093 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
11094 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11095 }
11096 return 0;
11097}
11098
11099unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11100 if (RetVT.SimpleTy != MVT::v16i16)
11101 return 0;
11102 if ((Subtarget->hasBF16()) && (Subtarget->hasVLX())) {
11103 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Z256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11104 }
11105 return 0;
11106}
11107
11108unsigned fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11109 if (RetVT.SimpleTy != MVT::v32i16)
11110 return 0;
11111 if ((Subtarget->hasBF16())) {
11112 return fastEmitInst_rr(X86::VCVTNE2PS2BF16Zrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11113 }
11114 return 0;
11115}
11116
11117unsigned fastEmit_X86ISD_CVTNE2PS2BF16_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11118 switch (VT.SimpleTy) {
11119 case MVT::v4f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11120 case MVT::v8f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11121 case MVT::v16f32: return fastEmit_X86ISD_CVTNE2PS2BF16_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11122 default: return 0;
11123 }
11124}
11125
11126// FastEmit functions for X86ISD::FADDS.
11127
11128unsigned fastEmit_X86ISD_FADDS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11129 if (RetVT.SimpleTy != MVT::v4f32)
11130 return 0;
11131 if ((Subtarget->hasAVX512())) {
11132 return fastEmitInst_rr(X86::VADDSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11133 }
11134 return 0;
11135}
11136
11137unsigned fastEmit_X86ISD_FADDS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11138 if (RetVT.SimpleTy != MVT::v2f64)
11139 return 0;
11140 if ((Subtarget->hasAVX512())) {
11141 return fastEmitInst_rr(X86::VADDSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11142 }
11143 return 0;
11144}
11145
11146unsigned fastEmit_X86ISD_FADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11147 switch (VT.SimpleTy) {
11148 case MVT::v4f32: return fastEmit_X86ISD_FADDS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11149 case MVT::v2f64: return fastEmit_X86ISD_FADDS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11150 default: return 0;
11151 }
11152}
11153
11154// FastEmit functions for X86ISD::FAND.
11155
11156unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11157 if (RetVT.SimpleTy != MVT::f128)
11158 return 0;
11159 if ((Subtarget->hasVLX())) {
11160 return fastEmitInst_rr(X86::VANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11161 }
11162 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11163 return fastEmitInst_rr(X86::VANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11164 }
11165 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11166 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11167 }
11168 return 0;
11169}
11170
11171unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11172 if (RetVT.SimpleTy != MVT::v4f32)
11173 return 0;
11174 return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11175}
11176
11177unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11178 switch (VT.SimpleTy) {
11179 case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11180 case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11181 default: return 0;
11182 }
11183}
11184
11185// FastEmit functions for X86ISD::FANDN.
11186
11187unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11188 if (RetVT.SimpleTy != MVT::v4f32)
11189 return 0;
11190 return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11191}
11192
11193unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11194 switch (VT.SimpleTy) {
11195 case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11196 default: return 0;
11197 }
11198}
11199
11200// FastEmit functions for X86ISD::FCMP.
11201
11202unsigned fastEmit_X86ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11203 if (RetVT.SimpleTy != MVT::i32)
11204 return 0;
11205 if ((Subtarget->hasAVX512())) {
11206 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11207 }
11208 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11209 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11210 }
11211 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11212 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11213 }
11214 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) {
11215 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11216 }
11217 return 0;
11218}
11219
11220unsigned fastEmit_X86ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11221 if (RetVT.SimpleTy != MVT::i32)
11222 return 0;
11223 if ((Subtarget->hasAVX512())) {
11224 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11225 }
11226 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11227 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11228 }
11229 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11230 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11231 }
11232 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) {
11233 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11234 }
11235 return 0;
11236}
11237
11238unsigned fastEmit_X86ISD_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11239 if (RetVT.SimpleTy != MVT::i32)
11240 return 0;
11241 if ((Subtarget->hasCMov())) {
11242 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11243 }
11244 return 0;
11245}
11246
11247unsigned fastEmit_X86ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11248 switch (VT.SimpleTy) {
11249 case MVT::f32: return fastEmit_X86ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11250 case MVT::f64: return fastEmit_X86ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11251 case MVT::f80: return fastEmit_X86ISD_FCMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11252 default: return 0;
11253 }
11254}
11255
11256// FastEmit functions for X86ISD::FDIVS.
11257
11258unsigned fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11259 if (RetVT.SimpleTy != MVT::v4f32)
11260 return 0;
11261 if ((Subtarget->hasAVX512())) {
11262 return fastEmitInst_rr(X86::VDIVSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11263 }
11264 return 0;
11265}
11266
11267unsigned fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11268 if (RetVT.SimpleTy != MVT::v2f64)
11269 return 0;
11270 if ((Subtarget->hasAVX512())) {
11271 return fastEmitInst_rr(X86::VDIVSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11272 }
11273 return 0;
11274}
11275
11276unsigned fastEmit_X86ISD_FDIVS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11277 switch (VT.SimpleTy) {
11278 case MVT::v4f32: return fastEmit_X86ISD_FDIVS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11279 case MVT::v2f64: return fastEmit_X86ISD_FDIVS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11280 default: return 0;
11281 }
11282}
11283
11284// FastEmit functions for X86ISD::FGETEXPS.
11285
11286unsigned fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11287 if (RetVT.SimpleTy != MVT::v4f32)
11288 return 0;
11289 if ((Subtarget->hasAVX512())) {
11290 return fastEmitInst_rr(X86::VGETEXPSSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11291 }
11292 return 0;
11293}
11294
11295unsigned fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11296 if (RetVT.SimpleTy != MVT::v2f64)
11297 return 0;
11298 if ((Subtarget->hasAVX512())) {
11299 return fastEmitInst_rr(X86::VGETEXPSDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11300 }
11301 return 0;
11302}
11303
11304unsigned fastEmit_X86ISD_FGETEXPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11305 switch (VT.SimpleTy) {
11306 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11307 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11308 default: return 0;
11309 }
11310}
11311
11312// FastEmit functions for X86ISD::FGETEXPS_SAE.
11313
11314unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11315 if (RetVT.SimpleTy != MVT::v4f32)
11316 return 0;
11317 if ((Subtarget->hasAVX512())) {
11318 return fastEmitInst_rr(X86::VGETEXPSSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11319 }
11320 return 0;
11321}
11322
11323unsigned fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11324 if (RetVT.SimpleTy != MVT::v2f64)
11325 return 0;
11326 if ((Subtarget->hasAVX512())) {
11327 return fastEmitInst_rr(X86::VGETEXPSDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11328 }
11329 return 0;
11330}
11331
11332unsigned fastEmit_X86ISD_FGETEXPS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11333 switch (VT.SimpleTy) {
11334 case MVT::v4f32: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11335 case MVT::v2f64: return fastEmit_X86ISD_FGETEXPS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11336 default: return 0;
11337 }
11338}
11339
11340// FastEmit functions for X86ISD::FHADD.
11341
11342unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11343 if (RetVT.SimpleTy != MVT::v4f32)
11344 return 0;
11345 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
11346 return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11347 }
11348 if ((Subtarget->hasAVX())) {
11349 return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11350 }
11351 return 0;
11352}
11353
11354unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11355 if (RetVT.SimpleTy != MVT::v8f32)
11356 return 0;
11357 if ((Subtarget->hasAVX())) {
11358 return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11359 }
11360 return 0;
11361}
11362
11363unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11364 if (RetVT.SimpleTy != MVT::v2f64)
11365 return 0;
11366 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
11367 return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11368 }
11369 if ((Subtarget->hasAVX())) {
11370 return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11371 }
11372 return 0;
11373}
11374
11375unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11376 if (RetVT.SimpleTy != MVT::v4f64)
11377 return 0;
11378 if ((Subtarget->hasAVX())) {
11379 return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11380 }
11381 return 0;
11382}
11383
11384unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11385 switch (VT.SimpleTy) {
11386 case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11387 case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11388 case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11389 case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11390 default: return 0;
11391 }
11392}
11393
11394// FastEmit functions for X86ISD::FHSUB.
11395
11396unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11397 if (RetVT.SimpleTy != MVT::v4f32)
11398 return 0;
11399 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
11400 return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11401 }
11402 if ((Subtarget->hasAVX())) {
11403 return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11404 }
11405 return 0;
11406}
11407
11408unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11409 if (RetVT.SimpleTy != MVT::v8f32)
11410 return 0;
11411 if ((Subtarget->hasAVX())) {
11412 return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11413 }
11414 return 0;
11415}
11416
11417unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11418 if (RetVT.SimpleTy != MVT::v2f64)
11419 return 0;
11420 if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
11421 return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11422 }
11423 if ((Subtarget->hasAVX())) {
11424 return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11425 }
11426 return 0;
11427}
11428
11429unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11430 if (RetVT.SimpleTy != MVT::v4f64)
11431 return 0;
11432 if ((Subtarget->hasAVX())) {
11433 return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11434 }
11435 return 0;
11436}
11437
11438unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11439 switch (VT.SimpleTy) {
11440 case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11441 case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11442 case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11443 case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11444 default: return 0;
11445 }
11446}
11447
11448// FastEmit functions for X86ISD::FMAX.
11449
11450unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11451 if (RetVT.SimpleTy != MVT::f32)
11452 return 0;
11453 if ((Subtarget->hasAVX512())) {
11454 return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11455 }
11456 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11457 return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11458 }
11459 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11460 return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11461 }
11462 return 0;
11463}
11464
11465unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11466 if (RetVT.SimpleTy != MVT::f64)
11467 return 0;
11468 if ((Subtarget->hasAVX512())) {
11469 return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11470 }
11471 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11472 return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11473 }
11474 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11475 return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11476 }
11477 return 0;
11478}
11479
11480unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11481 if (RetVT.SimpleTy != MVT::v4f32)
11482 return 0;
11483 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11484 return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11485 }
11486 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11487 return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11488 }
11489 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11490 return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11491 }
11492 return 0;
11493}
11494
11495unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11496 if (RetVT.SimpleTy != MVT::v8f32)
11497 return 0;
11498 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11499 return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11500 }
11501 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11502 return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11503 }
11504 return 0;
11505}
11506
11507unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11508 if (RetVT.SimpleTy != MVT::v16f32)
11509 return 0;
11510 if ((Subtarget->hasAVX512())) {
11511 return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11512 }
11513 return 0;
11514}
11515
11516unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11517 if (RetVT.SimpleTy != MVT::v2f64)
11518 return 0;
11519 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11520 return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11521 }
11522 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11523 return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11524 }
11525 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11526 return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11527 }
11528 return 0;
11529}
11530
11531unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11532 if (RetVT.SimpleTy != MVT::v4f64)
11533 return 0;
11534 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11535 return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11536 }
11537 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11538 return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11539 }
11540 return 0;
11541}
11542
11543unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11544 if (RetVT.SimpleTy != MVT::v8f64)
11545 return 0;
11546 if ((Subtarget->hasAVX512())) {
11547 return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11548 }
11549 return 0;
11550}
11551
11552unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11553 switch (VT.SimpleTy) {
11554 case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11555 case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11556 case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11557 case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11558 case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11559 case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11560 case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11561 case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11562 default: return 0;
11563 }
11564}
11565
11566// FastEmit functions for X86ISD::FMAXC.
11567
11568unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11569 if (RetVT.SimpleTy != MVT::f32)
11570 return 0;
11571 if ((Subtarget->hasAVX512())) {
11572 return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11573 }
11574 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11575 return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11576 }
11577 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11578 return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11579 }
11580 return 0;
11581}
11582
11583unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11584 if (RetVT.SimpleTy != MVT::f64)
11585 return 0;
11586 if ((Subtarget->hasAVX512())) {
11587 return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11588 }
11589 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11590 return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11591 }
11592 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11593 return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11594 }
11595 return 0;
11596}
11597
11598unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11599 if (RetVT.SimpleTy != MVT::v4f32)
11600 return 0;
11601 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11602 return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11603 }
11604 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11605 return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11606 }
11607 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11608 return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11609 }
11610 return 0;
11611}
11612
11613unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11614 if (RetVT.SimpleTy != MVT::v8f32)
11615 return 0;
11616 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11617 return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11618 }
11619 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11620 return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11621 }
11622 return 0;
11623}
11624
11625unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11626 if (RetVT.SimpleTy != MVT::v16f32)
11627 return 0;
11628 if ((Subtarget->hasAVX512())) {
11629 return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11630 }
11631 return 0;
11632}
11633
11634unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11635 if (RetVT.SimpleTy != MVT::v2f64)
11636 return 0;
11637 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11638 return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11639 }
11640 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11641 return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11642 }
11643 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11644 return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11645 }
11646 return 0;
11647}
11648
11649unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11650 if (RetVT.SimpleTy != MVT::v4f64)
11651 return 0;
11652 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11653 return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11654 }
11655 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11656 return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11657 }
11658 return 0;
11659}
11660
11661unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11662 if (RetVT.SimpleTy != MVT::v8f64)
11663 return 0;
11664 if ((Subtarget->hasAVX512())) {
11665 return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11666 }
11667 return 0;
11668}
11669
11670unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11671 switch (VT.SimpleTy) {
11672 case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11673 case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11674 case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11675 case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11676 case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11677 case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11678 case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11679 case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11680 default: return 0;
11681 }
11682}
11683
11684// FastEmit functions for X86ISD::FMAXS.
11685
11686unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11687 if (RetVT.SimpleTy != MVT::v4f32)
11688 return 0;
11689 if ((Subtarget->hasAVX512())) {
11690 return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11691 }
11692 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11693 return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11694 }
11695 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11696 return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11697 }
11698 return 0;
11699}
11700
11701unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11702 if (RetVT.SimpleTy != MVT::v2f64)
11703 return 0;
11704 if ((Subtarget->hasAVX512())) {
11705 return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11706 }
11707 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11708 return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11709 }
11710 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11711 return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11712 }
11713 return 0;
11714}
11715
11716unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11717 switch (VT.SimpleTy) {
11718 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11719 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11720 default: return 0;
11721 }
11722}
11723
11724// FastEmit functions for X86ISD::FMAXS_SAE.
11725
11726unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11727 if (RetVT.SimpleTy != MVT::v4f32)
11728 return 0;
11729 if ((Subtarget->hasAVX512())) {
11730 return fastEmitInst_rr(X86::VMAXSSZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11731 }
11732 return 0;
11733}
11734
11735unsigned fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11736 if (RetVT.SimpleTy != MVT::v2f64)
11737 return 0;
11738 if ((Subtarget->hasAVX512())) {
11739 return fastEmitInst_rr(X86::VMAXSDZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11740 }
11741 return 0;
11742}
11743
11744unsigned fastEmit_X86ISD_FMAXS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11745 switch (VT.SimpleTy) {
11746 case MVT::v4f32: return fastEmit_X86ISD_FMAXS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11747 case MVT::v2f64: return fastEmit_X86ISD_FMAXS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11748 default: return 0;
11749 }
11750}
11751
11752// FastEmit functions for X86ISD::FMAX_SAE.
11753
11754unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11755 if (RetVT.SimpleTy != MVT::v16f32)
11756 return 0;
11757 if ((Subtarget->hasAVX512())) {
11758 return fastEmitInst_rr(X86::VMAXPSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11759 }
11760 return 0;
11761}
11762
11763unsigned fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11764 if (RetVT.SimpleTy != MVT::v8f64)
11765 return 0;
11766 if ((Subtarget->hasAVX512())) {
11767 return fastEmitInst_rr(X86::VMAXPDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11768 }
11769 return 0;
11770}
11771
11772unsigned fastEmit_X86ISD_FMAX_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11773 switch (VT.SimpleTy) {
11774 case MVT::v16f32: return fastEmit_X86ISD_FMAX_SAE_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11775 case MVT::v8f64: return fastEmit_X86ISD_FMAX_SAE_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11776 default: return 0;
11777 }
11778}
11779
11780// FastEmit functions for X86ISD::FMIN.
11781
11782unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11783 if (RetVT.SimpleTy != MVT::f32)
11784 return 0;
11785 if ((Subtarget->hasAVX512())) {
11786 return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11787 }
11788 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11789 return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11790 }
11791 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11792 return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11793 }
11794 return 0;
11795}
11796
11797unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11798 if (RetVT.SimpleTy != MVT::f64)
11799 return 0;
11800 if ((Subtarget->hasAVX512())) {
11801 return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11802 }
11803 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11804 return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11805 }
11806 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11807 return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11808 }
11809 return 0;
11810}
11811
11812unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11813 if (RetVT.SimpleTy != MVT::v4f32)
11814 return 0;
11815 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11816 return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11817 }
11818 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11819 return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11820 }
11821 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11822 return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11823 }
11824 return 0;
11825}
11826
11827unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11828 if (RetVT.SimpleTy != MVT::v8f32)
11829 return 0;
11830 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11831 return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11832 }
11833 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11834 return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11835 }
11836 return 0;
11837}
11838
11839unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11840 if (RetVT.SimpleTy != MVT::v16f32)
11841 return 0;
11842 if ((Subtarget->hasAVX512())) {
11843 return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11844 }
11845 return 0;
11846}
11847
11848unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11849 if (RetVT.SimpleTy != MVT::v2f64)
11850 return 0;
11851 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11852 return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11853 }
11854 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11855 return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11856 }
11857 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11858 return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11859 }
11860 return 0;
11861}
11862
11863unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11864 if (RetVT.SimpleTy != MVT::v4f64)
11865 return 0;
11866 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11867 return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11868 }
11869 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11870 return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11871 }
11872 return 0;
11873}
11874
11875unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11876 if (RetVT.SimpleTy != MVT::v8f64)
11877 return 0;
11878 if ((Subtarget->hasAVX512())) {
11879 return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11880 }
11881 return 0;
11882}
11883
11884unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11885 switch (VT.SimpleTy) {
11886 case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11887 case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11888 case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11889 case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11890 case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11891 case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11892 case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11893 case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
11894 default: return 0;
11895 }
11896}
11897
11898// FastEmit functions for X86ISD::FMINC.
11899
11900unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11901 if (RetVT.SimpleTy != MVT::f32)
11902 return 0;
11903 if ((Subtarget->hasAVX512())) {
11904 return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11905 }
11906 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11907 return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11908 }
11909 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11910 return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11911 }
11912 return 0;
11913}
11914
11915unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11916 if (RetVT.SimpleTy != MVT::f64)
11917 return 0;
11918 if ((Subtarget->hasAVX512())) {
11919 return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11920 }
11921 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11922 return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11923 }
11924 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
11925 return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11926 }
11927 return 0;
11928}
11929
11930unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11931 if (RetVT.SimpleTy != MVT::v4f32)
11932 return 0;
11933 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11934 return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11935 }
11936 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
11937 return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11938 }
11939 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11940 return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11941 }
11942 return 0;
11943}
11944
11945unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11946 if (RetVT.SimpleTy != MVT::v8f32)
11947 return 0;
11948 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11949 return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11950 }
11951 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11952 return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11953 }
11954 return 0;
11955}
11956
11957unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11958 if (RetVT.SimpleTy != MVT::v16f32)
11959 return 0;
11960 if ((Subtarget->hasAVX512())) {
11961 return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11962 }
11963 return 0;
11964}
11965
11966unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11967 if (RetVT.SimpleTy != MVT::v2f64)
11968 return 0;
11969 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11970 return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11971 }
11972 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
11973 return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11974 }
11975 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11976 return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11977 }
11978 return 0;
11979}
11980
11981unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11982 if (RetVT.SimpleTy != MVT::v4f64)
11983 return 0;
11984 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
11985 return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11986 }
11987 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
11988 return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11989 }
11990 return 0;
11991}
11992
11993unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
11994 if (RetVT.SimpleTy != MVT::v8f64)
11995 return 0;
11996 if ((Subtarget->hasAVX512())) {
11997 return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
11998 }
11999 return 0;
12000}
12001
12002unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12003 switch (VT.SimpleTy) {
12004 case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12005 case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12006 case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12007 case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12008 case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12009 case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12010 case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12011 case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12012 default: return 0;
12013 }
12014}
12015
12016// FastEmit functions for X86ISD::FMINS.
12017
12018unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12019 if (RetVT.SimpleTy != MVT::v4f32)
12020 return 0;
12021 if ((Subtarget->hasAVX512())) {
12022 return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12023 }
12024 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12025 return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12026 }
12027 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12028 return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12029 }
12030 return 0;
12031}
12032
12033unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12034 if (RetVT.SimpleTy != MVT::v2f64)
12035 return 0;
12036 if ((Subtarget->hasAVX512())) {
12037 return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12038 }
12039 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12040 return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12041 }
12042 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12043 return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12044 }
12045 return 0;
12046}
12047
12048unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12049 switch (VT.SimpleTy) {
12050 case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12051 case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12052 default: return 0;
12053 }
12054}
12055
12056// FastEmit functions for X86ISD::FMINS_SAE.
12057
12058unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12059 if (RetVT.SimpleTy != MVT::v4f32)
12060 return 0;
12061 if ((Subtarget->hasAVX512())) {
12062 return fastEmitInst_rr(X86::VMINSSZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12063 }
12064 return 0;
12065}
12066
12067unsigned fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12068 if (RetVT.SimpleTy != MVT::v2f64)
12069 return 0;
12070 if ((Subtarget->hasAVX512())) {
12071 return fastEmitInst_rr(X86::VMINSDZrrb_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12072 }
12073 return 0;
12074}
12075
12076unsigned fastEmit_X86ISD_FMINS_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12077 switch (VT.SimpleTy) {
12078 case MVT::v4f32: return fastEmit_X86ISD_FMINS_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12079 case MVT::v2f64: return fastEmit_X86ISD_FMINS_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12080 default: return 0;
12081 }
12082}
12083
12084// FastEmit functions for X86ISD::FMIN_SAE.
12085
12086unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12087 if (RetVT.SimpleTy != MVT::v16f32)
12088 return 0;
12089 if ((Subtarget->hasAVX512())) {
12090 return fastEmitInst_rr(X86::VMINPSZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12091 }
12092 return 0;
12093}
12094
12095unsigned fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12096 if (RetVT.SimpleTy != MVT::v8f64)
12097 return 0;
12098 if ((Subtarget->hasAVX512())) {
12099 return fastEmitInst_rr(X86::VMINPDZrrb, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12100 }
12101 return 0;
12102}
12103
12104unsigned fastEmit_X86ISD_FMIN_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12105 switch (VT.SimpleTy) {
12106 case MVT::v16f32: return fastEmit_X86ISD_FMIN_SAE_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12107 case MVT::v8f64: return fastEmit_X86ISD_FMIN_SAE_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12108 default: return 0;
12109 }
12110}
12111
12112// FastEmit functions for X86ISD::FMULS.
12113
12114unsigned fastEmit_X86ISD_FMULS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12115 if (RetVT.SimpleTy != MVT::v4f32)
12116 return 0;
12117 if ((Subtarget->hasAVX512())) {
12118 return fastEmitInst_rr(X86::VMULSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12119 }
12120 return 0;
12121}
12122
12123unsigned fastEmit_X86ISD_FMULS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12124 if (RetVT.SimpleTy != MVT::v2f64)
12125 return 0;
12126 if ((Subtarget->hasAVX512())) {
12127 return fastEmitInst_rr(X86::VMULSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12128 }
12129 return 0;
12130}
12131
12132unsigned fastEmit_X86ISD_FMULS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12133 switch (VT.SimpleTy) {
12134 case MVT::v4f32: return fastEmit_X86ISD_FMULS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12135 case MVT::v2f64: return fastEmit_X86ISD_FMULS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12136 default: return 0;
12137 }
12138}
12139
12140// FastEmit functions for X86ISD::FOR.
12141
12142unsigned fastEmit_X86ISD_FOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12143 if (RetVT.SimpleTy != MVT::f128)
12144 return 0;
12145 if ((Subtarget->hasVLX())) {
12146 return fastEmitInst_rr(X86::VORPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12147 }
12148 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12149 return fastEmitInst_rr(X86::VORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12150 }
12151 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12152 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12153 }
12154 return 0;
12155}
12156
12157unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12158 if (RetVT.SimpleTy != MVT::v4f32)
12159 return 0;
12160 return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12161}
12162
12163unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12164 switch (VT.SimpleTy) {
12165 case MVT::f128: return fastEmit_X86ISD_FOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12166 case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12167 default: return 0;
12168 }
12169}
12170
12171// FastEmit functions for X86ISD::FSQRTS.
12172
12173unsigned fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12174 if (RetVT.SimpleTy != MVT::v4f32)
12175 return 0;
12176 if ((Subtarget->hasAVX512())) {
12177 return fastEmitInst_rr(X86::VSQRTSSZr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12178 }
12179 return 0;
12180}
12181
12182unsigned fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12183 if (RetVT.SimpleTy != MVT::v2f64)
12184 return 0;
12185 if ((Subtarget->hasAVX512())) {
12186 return fastEmitInst_rr(X86::VSQRTSDZr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12187 }
12188 return 0;
12189}
12190
12191unsigned fastEmit_X86ISD_FSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12192 switch (VT.SimpleTy) {
12193 case MVT::v4f32: return fastEmit_X86ISD_FSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12194 case MVT::v2f64: return fastEmit_X86ISD_FSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12195 default: return 0;
12196 }
12197}
12198
12199// FastEmit functions for X86ISD::FSUBS.
12200
12201unsigned fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12202 if (RetVT.SimpleTy != MVT::v4f32)
12203 return 0;
12204 if ((Subtarget->hasAVX512())) {
12205 return fastEmitInst_rr(X86::VSUBSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12206 }
12207 return 0;
12208}
12209
12210unsigned fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12211 if (RetVT.SimpleTy != MVT::v2f64)
12212 return 0;
12213 if ((Subtarget->hasAVX512())) {
12214 return fastEmitInst_rr(X86::VSUBSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12215 }
12216 return 0;
12217}
12218
12219unsigned fastEmit_X86ISD_FSUBS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12220 switch (VT.SimpleTy) {
12221 case MVT::v4f32: return fastEmit_X86ISD_FSUBS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12222 case MVT::v2f64: return fastEmit_X86ISD_FSUBS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12223 default: return 0;
12224 }
12225}
12226
12227// FastEmit functions for X86ISD::FXOR.
12228
12229unsigned fastEmit_X86ISD_FXOR_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12230 if (RetVT.SimpleTy != MVT::f128)
12231 return 0;
12232 if ((Subtarget->hasVLX())) {
12233 return fastEmitInst_rr(X86::VXORPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12234 }
12235 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
12236 return fastEmitInst_rr(X86::VXORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12237 }
12238 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12239 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12240 }
12241 return 0;
12242}
12243
12244unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12245 if (RetVT.SimpleTy != MVT::v4f32)
12246 return 0;
12247 return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12248}
12249
12250unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12251 switch (VT.SimpleTy) {
12252 case MVT::f128: return fastEmit_X86ISD_FXOR_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12253 case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12254 default: return 0;
12255 }
12256}
12257
12258// FastEmit functions for X86ISD::GF2P8MULB.
12259
12260unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12261 if (RetVT.SimpleTy != MVT::v16i8)
12262 return 0;
12263 if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
12264 return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12265 }
12266 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12267 return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12268 }
12269 if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12270 return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12271 }
12272 return 0;
12273}
12274
12275unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12276 if (RetVT.SimpleTy != MVT::v32i8)
12277 return 0;
12278 if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
12279 return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12280 }
12281 if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12282 return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12283 }
12284 return 0;
12285}
12286
12287unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12288 if (RetVT.SimpleTy != MVT::v64i8)
12289 return 0;
12290 if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) {
12291 return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12292 }
12293 return 0;
12294}
12295
12296unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12297 switch (VT.SimpleTy) {
12298 case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12299 case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12300 case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12301 default: return 0;
12302 }
12303}
12304
12305// FastEmit functions for X86ISD::HADD.
12306
12307unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12308 if (RetVT.SimpleTy != MVT::v8i16)
12309 return 0;
12310 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
12311 return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12312 }
12313 if ((Subtarget->hasAVX())) {
12314 return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12315 }
12316 return 0;
12317}
12318
12319unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12320 if (RetVT.SimpleTy != MVT::v16i16)
12321 return 0;
12322 if ((Subtarget->hasAVX2())) {
12323 return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12324 }
12325 return 0;
12326}
12327
12328unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12329 if (RetVT.SimpleTy != MVT::v4i32)
12330 return 0;
12331 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
12332 return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12333 }
12334 if ((Subtarget->hasAVX())) {
12335 return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12336 }
12337 return 0;
12338}
12339
12340unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12341 if (RetVT.SimpleTy != MVT::v8i32)
12342 return 0;
12343 if ((Subtarget->hasAVX2())) {
12344 return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12345 }
12346 return 0;
12347}
12348
12349unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12350 switch (VT.SimpleTy) {
12351 case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12352 case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12353 case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12354 case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12355 default: return 0;
12356 }
12357}
12358
12359// FastEmit functions for X86ISD::HSUB.
12360
12361unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12362 if (RetVT.SimpleTy != MVT::v8i16)
12363 return 0;
12364 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
12365 return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12366 }
12367 if ((Subtarget->hasAVX())) {
12368 return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12369 }
12370 return 0;
12371}
12372
12373unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12374 if (RetVT.SimpleTy != MVT::v16i16)
12375 return 0;
12376 if ((Subtarget->hasAVX2())) {
12377 return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12378 }
12379 return 0;
12380}
12381
12382unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12383 if (RetVT.SimpleTy != MVT::v4i32)
12384 return 0;
12385 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
12386 return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12387 }
12388 if ((Subtarget->hasAVX())) {
12389 return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12390 }
12391 return 0;
12392}
12393
12394unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12395 if (RetVT.SimpleTy != MVT::v8i32)
12396 return 0;
12397 if ((Subtarget->hasAVX2())) {
12398 return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12399 }
12400 return 0;
12401}
12402
12403unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12404 switch (VT.SimpleTy) {
12405 case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12406 case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12407 case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12408 case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12409 default: return 0;
12410 }
12411}
12412
12413// FastEmit functions for X86ISD::KADD.
12414
12415unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12416 if (RetVT.SimpleTy != MVT::v8i1)
12417 return 0;
12418 if ((Subtarget->hasDQI())) {
12419 return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12420 }
12421 return 0;
12422}
12423
12424unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12425 if (RetVT.SimpleTy != MVT::v16i1)
12426 return 0;
12427 if ((Subtarget->hasDQI())) {
12428 return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12429 }
12430 return 0;
12431}
12432
12433unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12434 if (RetVT.SimpleTy != MVT::v32i1)
12435 return 0;
12436 if ((Subtarget->hasBWI())) {
12437 return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12438 }
12439 return 0;
12440}
12441
12442unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12443 if (RetVT.SimpleTy != MVT::v64i1)
12444 return 0;
12445 if ((Subtarget->hasBWI())) {
12446 return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12447 }
12448 return 0;
12449}
12450
12451unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12452 switch (VT.SimpleTy) {
12453 case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12454 case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12455 case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12456 case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12457 default: return 0;
12458 }
12459}
12460
12461// FastEmit functions for X86ISD::KORTEST.
12462
12463unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12464 if (RetVT.SimpleTy != MVT::i32)
12465 return 0;
12466 if ((Subtarget->hasDQI())) {
12467 return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12468 }
12469 return 0;
12470}
12471
12472unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12473 if (RetVT.SimpleTy != MVT::i32)
12474 return 0;
12475 if ((Subtarget->hasAVX512())) {
12476 return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12477 }
12478 return 0;
12479}
12480
12481unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12482 if (RetVT.SimpleTy != MVT::i32)
12483 return 0;
12484 if ((Subtarget->hasBWI())) {
12485 return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12486 }
12487 return 0;
12488}
12489
12490unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12491 if (RetVT.SimpleTy != MVT::i32)
12492 return 0;
12493 if ((Subtarget->hasBWI())) {
12494 return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12495 }
12496 return 0;
12497}
12498
12499unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12500 switch (VT.SimpleTy) {
12501 case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12502 case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12503 case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12504 case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12505 default: return 0;
12506 }
12507}
12508
12509// FastEmit functions for X86ISD::KTEST.
12510
12511unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12512 if (RetVT.SimpleTy != MVT::i32)
12513 return 0;
12514 if ((Subtarget->hasDQI())) {
12515 return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12516 }
12517 return 0;
12518}
12519
12520unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12521 if (RetVT.SimpleTy != MVT::i32)
12522 return 0;
12523 if ((Subtarget->hasDQI())) {
12524 return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12525 }
12526 return 0;
12527}
12528
12529unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12530 if (RetVT.SimpleTy != MVT::i32)
12531 return 0;
12532 if ((Subtarget->hasBWI())) {
12533 return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12534 }
12535 return 0;
12536}
12537
12538unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12539 if (RetVT.SimpleTy != MVT::i32)
12540 return 0;
12541 if ((Subtarget->hasBWI())) {
12542 return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12543 }
12544 return 0;
12545}
12546
12547unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12548 switch (VT.SimpleTy) {
12549 case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12550 case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12551 case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12552 case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12553 default: return 0;
12554 }
12555}
12556
12557// FastEmit functions for X86ISD::MOVHLPS.
12558
12559unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12560 if (RetVT.SimpleTy != MVT::v4f32)
12561 return 0;
12562 if ((Subtarget->hasAVX512())) {
12563 return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12564 }
12565 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12566 return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12567 }
12568 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12569 return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12570 }
12571 return 0;
12572}
12573
12574unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12575 switch (VT.SimpleTy) {
12576 case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12577 default: return 0;
12578 }
12579}
12580
12581// FastEmit functions for X86ISD::MOVLHPS.
12582
12583unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12584 if (RetVT.SimpleTy != MVT::v4f32)
12585 return 0;
12586 if ((Subtarget->hasAVX512())) {
12587 return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12588 }
12589 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12590 return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12591 }
12592 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12593 return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12594 }
12595 return 0;
12596}
12597
12598unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12599 switch (VT.SimpleTy) {
12600 case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12601 default: return 0;
12602 }
12603}
12604
12605// FastEmit functions for X86ISD::MOVSD.
12606
12607unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12608 if (RetVT.SimpleTy != MVT::v2f64)
12609 return 0;
12610 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
12611 return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12612 }
12613 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12614 return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12615 }
12616 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12617 return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12618 }
12619 return 0;
12620}
12621
12622unsigned fastEmit_X86ISD_MOVSD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12623 switch (VT.SimpleTy) {
12624 case MVT::v2f64: return fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12625 default: return 0;
12626 }
12627}
12628
12629// FastEmit functions for X86ISD::MOVSS.
12630
12631unsigned fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12632 if (RetVT.SimpleTy != MVT::v4f32)
12633 return 0;
12634 if ((Subtarget->hasAVX512()) && (shouldOptForSize(MF))) {
12635 return fastEmitInst_rr(X86::VMOVSSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12636 }
12637 if ((shouldOptForSize(MF) || !Subtarget->hasSSE41()) && (Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
12638 return fastEmitInst_rr(X86::MOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12639 }
12640 if ((shouldOptForSize(MF)) && (Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
12641 return fastEmitInst_rr(X86::VMOVSSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12642 }
12643 return 0;
12644}
12645
12646unsigned fastEmit_X86ISD_MOVSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12647 switch (VT.SimpleTy) {
12648 case MVT::v4f32: return fastEmit_X86ISD_MOVSS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12649 default: return 0;
12650 }
12651}
12652
12653// FastEmit functions for X86ISD::MULHRS.
12654
12655unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12656 if (RetVT.SimpleTy != MVT::v8i16)
12657 return 0;
12658 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12659 return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12660 }
12661 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
12662 return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12663 }
12664 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12665 return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12666 }
12667 return 0;
12668}
12669
12670unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12671 if (RetVT.SimpleTy != MVT::v16i16)
12672 return 0;
12673 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12674 return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12675 }
12676 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12677 return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12678 }
12679 return 0;
12680}
12681
12682unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12683 if (RetVT.SimpleTy != MVT::v32i16)
12684 return 0;
12685 if ((Subtarget->hasBWI())) {
12686 return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12687 }
12688 return 0;
12689}
12690
12691unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12692 switch (VT.SimpleTy) {
12693 case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12694 case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12695 case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12696 default: return 0;
12697 }
12698}
12699
12700// FastEmit functions for X86ISD::MULTISHIFT.
12701
12702unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12703 if (RetVT.SimpleTy != MVT::v16i8)
12704 return 0;
12705 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
12706 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12707 }
12708 return 0;
12709}
12710
12711unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12712 if (RetVT.SimpleTy != MVT::v32i8)
12713 return 0;
12714 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
12715 return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12716 }
12717 return 0;
12718}
12719
12720unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12721 if (RetVT.SimpleTy != MVT::v64i8)
12722 return 0;
12723 if ((Subtarget->hasVBMI())) {
12724 return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12725 }
12726 return 0;
12727}
12728
12729unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12730 switch (VT.SimpleTy) {
12731 case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12732 case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12733 case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12734 default: return 0;
12735 }
12736}
12737
12738// FastEmit functions for X86ISD::PACKSS.
12739
12740unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12741 if (RetVT.SimpleTy != MVT::v16i8)
12742 return 0;
12743 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12744 return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12745 }
12746 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12747 return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12748 }
12749 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12750 return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12751 }
12752 return 0;
12753}
12754
12755unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12756 if (RetVT.SimpleTy != MVT::v32i8)
12757 return 0;
12758 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12759 return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12760 }
12761 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12762 return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12763 }
12764 return 0;
12765}
12766
12767unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12768 if (RetVT.SimpleTy != MVT::v64i8)
12769 return 0;
12770 if ((Subtarget->hasBWI())) {
12771 return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12772 }
12773 return 0;
12774}
12775
12776unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12777 if (RetVT.SimpleTy != MVT::v8i16)
12778 return 0;
12779 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12780 return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12781 }
12782 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12783 return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12784 }
12785 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12786 return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12787 }
12788 return 0;
12789}
12790
12791unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12792 if (RetVT.SimpleTy != MVT::v16i16)
12793 return 0;
12794 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12795 return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12796 }
12797 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12798 return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12799 }
12800 return 0;
12801}
12802
12803unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12804 if (RetVT.SimpleTy != MVT::v32i16)
12805 return 0;
12806 if ((Subtarget->hasBWI())) {
12807 return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12808 }
12809 return 0;
12810}
12811
12812unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12813 switch (VT.SimpleTy) {
12814 case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12815 case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12816 case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12817 case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12818 case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12819 case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12820 default: return 0;
12821 }
12822}
12823
12824// FastEmit functions for X86ISD::PACKUS.
12825
12826unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12827 if (RetVT.SimpleTy != MVT::v16i8)
12828 return 0;
12829 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12830 return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12831 }
12832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12833 return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12834 }
12835 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12836 return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12837 }
12838 return 0;
12839}
12840
12841unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12842 if (RetVT.SimpleTy != MVT::v32i8)
12843 return 0;
12844 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12845 return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12846 }
12847 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12848 return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12849 }
12850 return 0;
12851}
12852
12853unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12854 if (RetVT.SimpleTy != MVT::v64i8)
12855 return 0;
12856 if ((Subtarget->hasBWI())) {
12857 return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12858 }
12859 return 0;
12860}
12861
12862unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12863 if (RetVT.SimpleTy != MVT::v8i16)
12864 return 0;
12865 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12866 return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12867 }
12868 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12869 return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12870 }
12871 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12872 return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12873 }
12874 return 0;
12875}
12876
12877unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12878 if (RetVT.SimpleTy != MVT::v16i16)
12879 return 0;
12880 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
12881 return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12882 }
12883 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
12884 return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12885 }
12886 return 0;
12887}
12888
12889unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12890 if (RetVT.SimpleTy != MVT::v32i16)
12891 return 0;
12892 if ((Subtarget->hasBWI())) {
12893 return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12894 }
12895 return 0;
12896}
12897
12898unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12899 switch (VT.SimpleTy) {
12900 case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12901 case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12902 case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12903 case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12904 case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12905 case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12906 default: return 0;
12907 }
12908}
12909
12910// FastEmit functions for X86ISD::PCMPEQ.
12911
12912unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12913 if (RetVT.SimpleTy != MVT::v16i8)
12914 return 0;
12915 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12916 return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12917 }
12918 if ((Subtarget->hasAVX()) && (true)) {
12919 return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12920 }
12921 return 0;
12922}
12923
12924unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12925 if (RetVT.SimpleTy != MVT::v32i8)
12926 return 0;
12927 if ((Subtarget->hasAVX2()) && (true)) {
12928 return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12929 }
12930 return 0;
12931}
12932
12933unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12934 if (RetVT.SimpleTy != MVT::v8i16)
12935 return 0;
12936 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12937 return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12938 }
12939 if ((Subtarget->hasAVX()) && (true)) {
12940 return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12941 }
12942 return 0;
12943}
12944
12945unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12946 if (RetVT.SimpleTy != MVT::v16i16)
12947 return 0;
12948 if ((Subtarget->hasAVX2()) && (true)) {
12949 return fastEmitInst_rr(X86::VPCMPEQWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12950 }
12951 return 0;
12952}
12953
12954unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12955 if (RetVT.SimpleTy != MVT::v4i32)
12956 return 0;
12957 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
12958 return fastEmitInst_rr(X86::PCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12959 }
12960 if ((Subtarget->hasAVX()) && (true)) {
12961 return fastEmitInst_rr(X86::VPCMPEQDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12962 }
12963 return 0;
12964}
12965
12966unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12967 if (RetVT.SimpleTy != MVT::v8i32)
12968 return 0;
12969 if ((Subtarget->hasAVX2()) && (true)) {
12970 return fastEmitInst_rr(X86::VPCMPEQDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12971 }
12972 return 0;
12973}
12974
12975unsigned fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12976 if (RetVT.SimpleTy != MVT::v2i64)
12977 return 0;
12978 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
12979 return fastEmitInst_rr(X86::PCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12980 }
12981 if ((Subtarget->hasAVX())) {
12982 return fastEmitInst_rr(X86::VPCMPEQQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12983 }
12984 return 0;
12985}
12986
12987unsigned fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12988 if (RetVT.SimpleTy != MVT::v4i64)
12989 return 0;
12990 if ((Subtarget->hasAVX2())) {
12991 return fastEmitInst_rr(X86::VPCMPEQQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
12992 }
12993 return 0;
12994}
12995
12996unsigned fastEmit_X86ISD_PCMPEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
12997 switch (VT.SimpleTy) {
12998 case MVT::v16i8: return fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
12999 case MVT::v32i8: return fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13000 case MVT::v8i16: return fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13001 case MVT::v16i16: return fastEmit_X86ISD_PCMPEQ_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13002 case MVT::v4i32: return fastEmit_X86ISD_PCMPEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13003 case MVT::v8i32: return fastEmit_X86ISD_PCMPEQ_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13004 case MVT::v2i64: return fastEmit_X86ISD_PCMPEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13005 case MVT::v4i64: return fastEmit_X86ISD_PCMPEQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13006 default: return 0;
13007 }
13008}
13009
13010// FastEmit functions for X86ISD::PCMPGT.
13011
13012unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13013 if (RetVT.SimpleTy != MVT::v16i8)
13014 return 0;
13015 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13016 return fastEmitInst_rr(X86::PCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13017 }
13018 if ((Subtarget->hasAVX()) && (true)) {
13019 return fastEmitInst_rr(X86::VPCMPGTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13020 }
13021 return 0;
13022}
13023
13024unsigned fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13025 if (RetVT.SimpleTy != MVT::v32i8)
13026 return 0;
13027 if ((Subtarget->hasAVX2()) && (true)) {
13028 return fastEmitInst_rr(X86::VPCMPGTBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13029 }
13030 return 0;
13031}
13032
13033unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13034 if (RetVT.SimpleTy != MVT::v8i16)
13035 return 0;
13036 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13037 return fastEmitInst_rr(X86::PCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13038 }
13039 if ((Subtarget->hasAVX()) && (true)) {
13040 return fastEmitInst_rr(X86::VPCMPGTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13041 }
13042 return 0;
13043}
13044
13045unsigned fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13046 if (RetVT.SimpleTy != MVT::v16i16)
13047 return 0;
13048 if ((Subtarget->hasAVX2()) && (true)) {
13049 return fastEmitInst_rr(X86::VPCMPGTWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13050 }
13051 return 0;
13052}
13053
13054unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13055 if (RetVT.SimpleTy != MVT::v4i32)
13056 return 0;
13057 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13058 return fastEmitInst_rr(X86::PCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13059 }
13060 if ((Subtarget->hasAVX()) && (true)) {
13061 return fastEmitInst_rr(X86::VPCMPGTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13062 }
13063 return 0;
13064}
13065
13066unsigned fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13067 if (RetVT.SimpleTy != MVT::v8i32)
13068 return 0;
13069 if ((Subtarget->hasAVX2()) && (true)) {
13070 return fastEmitInst_rr(X86::VPCMPGTDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13071 }
13072 return 0;
13073}
13074
13075unsigned fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13076 if (RetVT.SimpleTy != MVT::v2i64)
13077 return 0;
13078 if ((Subtarget->hasSSE42() && !Subtarget->hasAVX())) {
13079 return fastEmitInst_rr(X86::PCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13080 }
13081 if ((Subtarget->hasAVX())) {
13082 return fastEmitInst_rr(X86::VPCMPGTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13083 }
13084 return 0;
13085}
13086
13087unsigned fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13088 if (RetVT.SimpleTy != MVT::v4i64)
13089 return 0;
13090 if ((Subtarget->hasAVX2())) {
13091 return fastEmitInst_rr(X86::VPCMPGTQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13092 }
13093 return 0;
13094}
13095
13096unsigned fastEmit_X86ISD_PCMPGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13097 switch (VT.SimpleTy) {
13098 case MVT::v16i8: return fastEmit_X86ISD_PCMPGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13099 case MVT::v32i8: return fastEmit_X86ISD_PCMPGT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13100 case MVT::v8i16: return fastEmit_X86ISD_PCMPGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13101 case MVT::v16i16: return fastEmit_X86ISD_PCMPGT_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13102 case MVT::v4i32: return fastEmit_X86ISD_PCMPGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13103 case MVT::v8i32: return fastEmit_X86ISD_PCMPGT_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13104 case MVT::v2i64: return fastEmit_X86ISD_PCMPGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13105 case MVT::v4i64: return fastEmit_X86ISD_PCMPGT_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13106 default: return 0;
13107 }
13108}
13109
13110// FastEmit functions for X86ISD::PDEP.
13111
13112unsigned fastEmit_X86ISD_PDEP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13113 if (RetVT.SimpleTy != MVT::i32)
13114 return 0;
13115 if ((Subtarget->hasBMI2())) {
13116 return fastEmitInst_rr(X86::PDEP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13117 }
13118 return 0;
13119}
13120
13121unsigned fastEmit_X86ISD_PDEP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13122 if (RetVT.SimpleTy != MVT::i64)
13123 return 0;
13124 if ((Subtarget->hasBMI2())) {
13125 return fastEmitInst_rr(X86::PDEP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13126 }
13127 return 0;
13128}
13129
13130unsigned fastEmit_X86ISD_PDEP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13131 switch (VT.SimpleTy) {
13132 case MVT::i32: return fastEmit_X86ISD_PDEP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13133 case MVT::i64: return fastEmit_X86ISD_PDEP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13134 default: return 0;
13135 }
13136}
13137
13138// FastEmit functions for X86ISD::PEXT.
13139
13140unsigned fastEmit_X86ISD_PEXT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13141 if (RetVT.SimpleTy != MVT::i32)
13142 return 0;
13143 if ((Subtarget->hasBMI2())) {
13144 return fastEmitInst_rr(X86::PEXT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13145 }
13146 return 0;
13147}
13148
13149unsigned fastEmit_X86ISD_PEXT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13150 if (RetVT.SimpleTy != MVT::i64)
13151 return 0;
13152 if ((Subtarget->hasBMI2())) {
13153 return fastEmitInst_rr(X86::PEXT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13154 }
13155 return 0;
13156}
13157
13158unsigned fastEmit_X86ISD_PEXT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13159 switch (VT.SimpleTy) {
13160 case MVT::i32: return fastEmit_X86ISD_PEXT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13161 case MVT::i64: return fastEmit_X86ISD_PEXT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13162 default: return 0;
13163 }
13164}
13165
13166// FastEmit functions for X86ISD::PMULDQ.
13167
13168unsigned fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13169 if (RetVT.SimpleTy != MVT::v2i64)
13170 return 0;
13171 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13172 return fastEmitInst_rr(X86::VPMULDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13173 }
13174 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13175 return fastEmitInst_rr(X86::PMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13176 }
13177 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13178 return fastEmitInst_rr(X86::VPMULDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13179 }
13180 return 0;
13181}
13182
13183unsigned fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13184 if (RetVT.SimpleTy != MVT::v4i64)
13185 return 0;
13186 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13187 return fastEmitInst_rr(X86::VPMULDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13188 }
13189 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13190 return fastEmitInst_rr(X86::VPMULDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13191 }
13192 return 0;
13193}
13194
13195unsigned fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13196 if (RetVT.SimpleTy != MVT::v8i64)
13197 return 0;
13198 if ((Subtarget->hasAVX512())) {
13199 return fastEmitInst_rr(X86::VPMULDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13200 }
13201 return 0;
13202}
13203
13204unsigned fastEmit_X86ISD_PMULDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13205 switch (VT.SimpleTy) {
13206 case MVT::v2i64: return fastEmit_X86ISD_PMULDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13207 case MVT::v4i64: return fastEmit_X86ISD_PMULDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13208 case MVT::v8i64: return fastEmit_X86ISD_PMULDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13209 default: return 0;
13210 }
13211}
13212
13213// FastEmit functions for X86ISD::PMULUDQ.
13214
13215unsigned fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13216 if (RetVT.SimpleTy != MVT::v2i64)
13217 return 0;
13218 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13219 return fastEmitInst_rr(X86::VPMULUDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13220 }
13221 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13222 return fastEmitInst_rr(X86::PMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13223 }
13224 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13225 return fastEmitInst_rr(X86::VPMULUDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13226 }
13227 return 0;
13228}
13229
13230unsigned fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13231 if (RetVT.SimpleTy != MVT::v4i64)
13232 return 0;
13233 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13234 return fastEmitInst_rr(X86::VPMULUDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13235 }
13236 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13237 return fastEmitInst_rr(X86::VPMULUDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13238 }
13239 return 0;
13240}
13241
13242unsigned fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13243 if (RetVT.SimpleTy != MVT::v8i64)
13244 return 0;
13245 if ((Subtarget->hasAVX512())) {
13246 return fastEmitInst_rr(X86::VPMULUDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13247 }
13248 return 0;
13249}
13250
13251unsigned fastEmit_X86ISD_PMULUDQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13252 switch (VT.SimpleTy) {
13253 case MVT::v2i64: return fastEmit_X86ISD_PMULUDQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13254 case MVT::v4i64: return fastEmit_X86ISD_PMULUDQ_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13255 case MVT::v8i64: return fastEmit_X86ISD_PMULUDQ_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13256 default: return 0;
13257 }
13258}
13259
13260// FastEmit functions for X86ISD::PSADBW.
13261
13262unsigned fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13263 if (RetVT.SimpleTy != MVT::v2i64)
13264 return 0;
13265 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13266 return fastEmitInst_rr(X86::VPSADBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13267 }
13268 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13269 return fastEmitInst_rr(X86::PSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13270 }
13271 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13272 return fastEmitInst_rr(X86::VPSADBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13273 }
13274 return 0;
13275}
13276
13277unsigned fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13278 if (RetVT.SimpleTy != MVT::v4i64)
13279 return 0;
13280 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13281 return fastEmitInst_rr(X86::VPSADBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13282 }
13283 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13284 return fastEmitInst_rr(X86::VPSADBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13285 }
13286 return 0;
13287}
13288
13289unsigned fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13290 if (RetVT.SimpleTy != MVT::v8i64)
13291 return 0;
13292 if ((Subtarget->hasBWI())) {
13293 return fastEmitInst_rr(X86::VPSADBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13294 }
13295 return 0;
13296}
13297
13298unsigned fastEmit_X86ISD_PSADBW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13299 switch (VT.SimpleTy) {
13300 case MVT::v16i8: return fastEmit_X86ISD_PSADBW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13301 case MVT::v32i8: return fastEmit_X86ISD_PSADBW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13302 case MVT::v64i8: return fastEmit_X86ISD_PSADBW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13303 default: return 0;
13304 }
13305}
13306
13307// FastEmit functions for X86ISD::PSHUFB.
13308
13309unsigned fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13310 if (RetVT.SimpleTy != MVT::v16i8)
13311 return 0;
13312 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13313 return fastEmitInst_rr(X86::VPSHUFBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13314 }
13315 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
13316 return fastEmitInst_rr(X86::PSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13317 }
13318 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13319 return fastEmitInst_rr(X86::VPSHUFBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13320 }
13321 return 0;
13322}
13323
13324unsigned fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13325 if (RetVT.SimpleTy != MVT::v32i8)
13326 return 0;
13327 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13328 return fastEmitInst_rr(X86::VPSHUFBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13329 }
13330 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13331 return fastEmitInst_rr(X86::VPSHUFBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13332 }
13333 return 0;
13334}
13335
13336unsigned fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13337 if (RetVT.SimpleTy != MVT::v64i8)
13338 return 0;
13339 if ((Subtarget->hasBWI())) {
13340 return fastEmitInst_rr(X86::VPSHUFBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13341 }
13342 return 0;
13343}
13344
13345unsigned fastEmit_X86ISD_PSHUFB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13346 switch (VT.SimpleTy) {
13347 case MVT::v16i8: return fastEmit_X86ISD_PSHUFB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13348 case MVT::v32i8: return fastEmit_X86ISD_PSHUFB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13349 case MVT::v64i8: return fastEmit_X86ISD_PSHUFB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13350 default: return 0;
13351 }
13352}
13353
13354// FastEmit functions for X86ISD::PTEST.
13355
13356unsigned fastEmit_X86ISD_PTEST_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13357 if (RetVT.SimpleTy != MVT::i32)
13358 return 0;
13359 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
13360 return fastEmitInst_rr(X86::PTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13361 }
13362 if ((Subtarget->hasAVX())) {
13363 return fastEmitInst_rr(X86::VPTESTrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13364 }
13365 return 0;
13366}
13367
13368unsigned fastEmit_X86ISD_PTEST_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13369 if (RetVT.SimpleTy != MVT::i32)
13370 return 0;
13371 if ((Subtarget->hasAVX())) {
13372 return fastEmitInst_rr(X86::VPTESTYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13373 }
13374 return 0;
13375}
13376
13377unsigned fastEmit_X86ISD_PTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13378 switch (VT.SimpleTy) {
13379 case MVT::v2i64: return fastEmit_X86ISD_PTEST_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13380 case MVT::v4i64: return fastEmit_X86ISD_PTEST_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13381 default: return 0;
13382 }
13383}
13384
13385// FastEmit functions for X86ISD::RCP14S.
13386
13387unsigned fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13388 if (RetVT.SimpleTy != MVT::v4f32)
13389 return 0;
13390 if ((Subtarget->hasAVX512())) {
13391 return fastEmitInst_rr(X86::VRCP14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13392 }
13393 return 0;
13394}
13395
13396unsigned fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13397 if (RetVT.SimpleTy != MVT::v2f64)
13398 return 0;
13399 if ((Subtarget->hasAVX512())) {
13400 return fastEmitInst_rr(X86::VRCP14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13401 }
13402 return 0;
13403}
13404
13405unsigned fastEmit_X86ISD_RCP14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13406 switch (VT.SimpleTy) {
13407 case MVT::v4f32: return fastEmit_X86ISD_RCP14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13408 case MVT::v2f64: return fastEmit_X86ISD_RCP14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13409 default: return 0;
13410 }
13411}
13412
13413// FastEmit functions for X86ISD::RCP28S.
13414
13415unsigned fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13416 if (RetVT.SimpleTy != MVT::v4f32)
13417 return 0;
13418 if ((Subtarget->hasERI())) {
13419 return fastEmitInst_rr(X86::VRCP28SSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13420 }
13421 return 0;
13422}
13423
13424unsigned fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13425 if (RetVT.SimpleTy != MVT::v2f64)
13426 return 0;
13427 if ((Subtarget->hasERI())) {
13428 return fastEmitInst_rr(X86::VRCP28SDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13429 }
13430 return 0;
13431}
13432
13433unsigned fastEmit_X86ISD_RCP28S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13434 switch (VT.SimpleTy) {
13435 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13436 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13437 default: return 0;
13438 }
13439}
13440
13441// FastEmit functions for X86ISD::RCP28S_SAE.
13442
13443unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13444 if (RetVT.SimpleTy != MVT::v4f32)
13445 return 0;
13446 if ((Subtarget->hasERI())) {
13447 return fastEmitInst_rr(X86::VRCP28SSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13448 }
13449 return 0;
13450}
13451
13452unsigned fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13453 if (RetVT.SimpleTy != MVT::v2f64)
13454 return 0;
13455 if ((Subtarget->hasERI())) {
13456 return fastEmitInst_rr(X86::VRCP28SDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13457 }
13458 return 0;
13459}
13460
13461unsigned fastEmit_X86ISD_RCP28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13462 switch (VT.SimpleTy) {
13463 case MVT::v4f32: return fastEmit_X86ISD_RCP28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13464 case MVT::v2f64: return fastEmit_X86ISD_RCP28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13465 default: return 0;
13466 }
13467}
13468
13469// FastEmit functions for X86ISD::RSQRT14S.
13470
13471unsigned fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13472 if (RetVT.SimpleTy != MVT::v4f32)
13473 return 0;
13474 if ((Subtarget->hasAVX512())) {
13475 return fastEmitInst_rr(X86::VRSQRT14SSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13476 }
13477 return 0;
13478}
13479
13480unsigned fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13481 if (RetVT.SimpleTy != MVT::v2f64)
13482 return 0;
13483 if ((Subtarget->hasAVX512())) {
13484 return fastEmitInst_rr(X86::VRSQRT14SDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13485 }
13486 return 0;
13487}
13488
13489unsigned fastEmit_X86ISD_RSQRT14S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13490 switch (VT.SimpleTy) {
13491 case MVT::v4f32: return fastEmit_X86ISD_RSQRT14S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13492 case MVT::v2f64: return fastEmit_X86ISD_RSQRT14S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13493 default: return 0;
13494 }
13495}
13496
13497// FastEmit functions for X86ISD::RSQRT28S.
13498
13499unsigned fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13500 if (RetVT.SimpleTy != MVT::v4f32)
13501 return 0;
13502 if ((Subtarget->hasERI())) {
13503 return fastEmitInst_rr(X86::VRSQRT28SSZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13504 }
13505 return 0;
13506}
13507
13508unsigned fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13509 if (RetVT.SimpleTy != MVT::v2f64)
13510 return 0;
13511 if ((Subtarget->hasERI())) {
13512 return fastEmitInst_rr(X86::VRSQRT28SDZr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13513 }
13514 return 0;
13515}
13516
13517unsigned fastEmit_X86ISD_RSQRT28S_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13518 switch (VT.SimpleTy) {
13519 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13520 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13521 default: return 0;
13522 }
13523}
13524
13525// FastEmit functions for X86ISD::RSQRT28S_SAE.
13526
13527unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13528 if (RetVT.SimpleTy != MVT::v4f32)
13529 return 0;
13530 if ((Subtarget->hasERI())) {
13531 return fastEmitInst_rr(X86::VRSQRT28SSZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13532 }
13533 return 0;
13534}
13535
13536unsigned fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13537 if (RetVT.SimpleTy != MVT::v2f64)
13538 return 0;
13539 if ((Subtarget->hasERI())) {
13540 return fastEmitInst_rr(X86::VRSQRT28SDZrb, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13541 }
13542 return 0;
13543}
13544
13545unsigned fastEmit_X86ISD_RSQRT28S_SAE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13546 switch (VT.SimpleTy) {
13547 case MVT::v4f32: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13548 case MVT::v2f64: return fastEmit_X86ISD_RSQRT28S_SAE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13549 default: return 0;
13550 }
13551}
13552
13553// FastEmit functions for X86ISD::SCALEF.
13554
13555unsigned fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13556 if (RetVT.SimpleTy != MVT::v4f32)
13557 return 0;
13558 if ((Subtarget->hasVLX())) {
13559 return fastEmitInst_rr(X86::VSCALEFPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13560 }
13561 return 0;
13562}
13563
13564unsigned fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13565 if (RetVT.SimpleTy != MVT::v8f32)
13566 return 0;
13567 if ((Subtarget->hasVLX())) {
13568 return fastEmitInst_rr(X86::VSCALEFPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13569 }
13570 return 0;
13571}
13572
13573unsigned fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13574 if (RetVT.SimpleTy != MVT::v16f32)
13575 return 0;
13576 if ((Subtarget->hasAVX512())) {
13577 return fastEmitInst_rr(X86::VSCALEFPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13578 }
13579 return 0;
13580}
13581
13582unsigned fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13583 if (RetVT.SimpleTy != MVT::v2f64)
13584 return 0;
13585 if ((Subtarget->hasVLX())) {
13586 return fastEmitInst_rr(X86::VSCALEFPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13587 }
13588 return 0;
13589}
13590
13591unsigned fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13592 if (RetVT.SimpleTy != MVT::v4f64)
13593 return 0;
13594 if ((Subtarget->hasVLX())) {
13595 return fastEmitInst_rr(X86::VSCALEFPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13596 }
13597 return 0;
13598}
13599
13600unsigned fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13601 if (RetVT.SimpleTy != MVT::v8f64)
13602 return 0;
13603 if ((Subtarget->hasAVX512())) {
13604 return fastEmitInst_rr(X86::VSCALEFPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13605 }
13606 return 0;
13607}
13608
13609unsigned fastEmit_X86ISD_SCALEF_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13610 switch (VT.SimpleTy) {
13611 case MVT::v4f32: return fastEmit_X86ISD_SCALEF_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13612 case MVT::v8f32: return fastEmit_X86ISD_SCALEF_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13613 case MVT::v16f32: return fastEmit_X86ISD_SCALEF_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13614 case MVT::v2f64: return fastEmit_X86ISD_SCALEF_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13615 case MVT::v4f64: return fastEmit_X86ISD_SCALEF_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13616 case MVT::v8f64: return fastEmit_X86ISD_SCALEF_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13617 default: return 0;
13618 }
13619}
13620
13621// FastEmit functions for X86ISD::SCALEFS.
13622
13623unsigned fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13624 if (RetVT.SimpleTy != MVT::v4f32)
13625 return 0;
13626 if ((Subtarget->hasAVX512())) {
13627 return fastEmitInst_rr(X86::VSCALEFSSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13628 }
13629 return 0;
13630}
13631
13632unsigned fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13633 if (RetVT.SimpleTy != MVT::v2f64)
13634 return 0;
13635 if ((Subtarget->hasAVX512())) {
13636 return fastEmitInst_rr(X86::VSCALEFSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13637 }
13638 return 0;
13639}
13640
13641unsigned fastEmit_X86ISD_SCALEFS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13642 switch (VT.SimpleTy) {
13643 case MVT::v4f32: return fastEmit_X86ISD_SCALEFS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13644 case MVT::v2f64: return fastEmit_X86ISD_SCALEFS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13645 default: return 0;
13646 }
13647}
13648
13649// FastEmit functions for X86ISD::STRICT_FCMP.
13650
13651unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13652 if (RetVT.SimpleTy != MVT::i32)
13653 return 0;
13654 if ((Subtarget->hasAVX512())) {
13655 return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13656 }
13657 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13658 return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13659 }
13660 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13661 return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13662 }
13663 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) {
13664 return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13665 }
13666 return 0;
13667}
13668
13669unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13670 if (RetVT.SimpleTy != MVT::i32)
13671 return 0;
13672 if ((Subtarget->hasAVX512())) {
13673 return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13674 }
13675 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13676 return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13677 }
13678 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13679 return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13680 }
13681 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) {
13682 return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13683 }
13684 return 0;
13685}
13686
13687unsigned fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13688 if (RetVT.SimpleTy != MVT::i32)
13689 return 0;
13690 if ((Subtarget->hasCMov())) {
13691 return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13692 }
13693 return 0;
13694}
13695
13696unsigned fastEmit_X86ISD_STRICT_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13697 switch (VT.SimpleTy) {
13698 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13699 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13700 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13701 default: return 0;
13702 }
13703}
13704
13705// FastEmit functions for X86ISD::STRICT_FCMPS.
13706
13707unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13708 if (RetVT.SimpleTy != MVT::i32)
13709 return 0;
13710 if ((Subtarget->hasAVX512())) {
13711 return fastEmitInst_rr(X86::VCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13712 }
13713 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13714 return fastEmitInst_rr(X86::COMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13715 }
13716 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13717 return fastEmitInst_rr(X86::VCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13718 }
13719 if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) {
13720 return fastEmitInst_rr(X86::COM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13721 }
13722 return 0;
13723}
13724
13725unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13726 if (RetVT.SimpleTy != MVT::i32)
13727 return 0;
13728 if ((Subtarget->hasAVX512())) {
13729 return fastEmitInst_rr(X86::VCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13730 }
13731 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13732 return fastEmitInst_rr(X86::COMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13733 }
13734 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13735 return fastEmitInst_rr(X86::VCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13736 }
13737 if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) {
13738 return fastEmitInst_rr(X86::COM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13739 }
13740 return 0;
13741}
13742
13743unsigned fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13744 if (RetVT.SimpleTy != MVT::i32)
13745 return 0;
13746 if ((Subtarget->hasCMov())) {
13747 return fastEmitInst_rr(X86::COM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13748 }
13749 return 0;
13750}
13751
13752unsigned fastEmit_X86ISD_STRICT_FCMPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13753 switch (VT.SimpleTy) {
13754 case MVT::f32: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13755 case MVT::f64: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13756 case MVT::f80: return fastEmit_X86ISD_STRICT_FCMPS_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13757 default: return 0;
13758 }
13759}
13760
13761// FastEmit functions for X86ISD::TESTP.
13762
13763unsigned fastEmit_X86ISD_TESTP_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13764 if (RetVT.SimpleTy != MVT::i32)
13765 return 0;
13766 if ((Subtarget->hasAVX())) {
13767 return fastEmitInst_rr(X86::VTESTPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13768 }
13769 return 0;
13770}
13771
13772unsigned fastEmit_X86ISD_TESTP_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13773 if (RetVT.SimpleTy != MVT::i32)
13774 return 0;
13775 if ((Subtarget->hasAVX())) {
13776 return fastEmitInst_rr(X86::VTESTPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13777 }
13778 return 0;
13779}
13780
13781unsigned fastEmit_X86ISD_TESTP_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13782 if (RetVT.SimpleTy != MVT::i32)
13783 return 0;
13784 if ((Subtarget->hasAVX())) {
13785 return fastEmitInst_rr(X86::VTESTPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13786 }
13787 return 0;
13788}
13789
13790unsigned fastEmit_X86ISD_TESTP_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13791 if (RetVT.SimpleTy != MVT::i32)
13792 return 0;
13793 if ((Subtarget->hasAVX())) {
13794 return fastEmitInst_rr(X86::VTESTPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13795 }
13796 return 0;
13797}
13798
13799unsigned fastEmit_X86ISD_TESTP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13800 switch (VT.SimpleTy) {
13801 case MVT::v4f32: return fastEmit_X86ISD_TESTP_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13802 case MVT::v8f32: return fastEmit_X86ISD_TESTP_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13803 case MVT::v2f64: return fastEmit_X86ISD_TESTP_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13804 case MVT::v4f64: return fastEmit_X86ISD_TESTP_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13805 default: return 0;
13806 }
13807}
13808
13809// FastEmit functions for X86ISD::UCOMI.
13810
13811unsigned fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13812 if (RetVT.SimpleTy != MVT::i32)
13813 return 0;
13814 if ((Subtarget->hasAVX512())) {
13815 return fastEmitInst_rr(X86::VUCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13816 }
13817 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
13818 return fastEmitInst_rr(X86::UCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13819 }
13820 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13821 return fastEmitInst_rr(X86::VUCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13822 }
13823 return 0;
13824}
13825
13826unsigned fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13827 if (RetVT.SimpleTy != MVT::i32)
13828 return 0;
13829 if ((Subtarget->hasAVX512())) {
13830 return fastEmitInst_rr(X86::VUCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13831 }
13832 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13833 return fastEmitInst_rr(X86::UCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13834 }
13835 if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
13836 return fastEmitInst_rr(X86::VUCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13837 }
13838 return 0;
13839}
13840
13841unsigned fastEmit_X86ISD_UCOMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13842 switch (VT.SimpleTy) {
13843 case MVT::v4f32: return fastEmit_X86ISD_UCOMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13844 case MVT::v2f64: return fastEmit_X86ISD_UCOMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
13845 default: return 0;
13846 }
13847}
13848
13849// FastEmit functions for X86ISD::UNPCKH.
13850
13851unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13852 if (RetVT.SimpleTy != MVT::v16i8)
13853 return 0;
13854 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13855 return fastEmitInst_rr(X86::VPUNPCKHBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13856 }
13857 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13858 return fastEmitInst_rr(X86::PUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13859 }
13860 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13861 return fastEmitInst_rr(X86::VPUNPCKHBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13862 }
13863 return 0;
13864}
13865
13866unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13867 if (RetVT.SimpleTy != MVT::v32i8)
13868 return 0;
13869 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13870 return fastEmitInst_rr(X86::VPUNPCKHBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13871 }
13872 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13873 return fastEmitInst_rr(X86::VPUNPCKHBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13874 }
13875 return 0;
13876}
13877
13878unsigned fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13879 if (RetVT.SimpleTy != MVT::v64i8)
13880 return 0;
13881 if ((Subtarget->hasBWI())) {
13882 return fastEmitInst_rr(X86::VPUNPCKHBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13883 }
13884 return 0;
13885}
13886
13887unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13888 if (RetVT.SimpleTy != MVT::v8i16)
13889 return 0;
13890 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13891 return fastEmitInst_rr(X86::VPUNPCKHWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13892 }
13893 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13894 return fastEmitInst_rr(X86::PUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13895 }
13896 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13897 return fastEmitInst_rr(X86::VPUNPCKHWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13898 }
13899 return 0;
13900}
13901
13902unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13903 if (RetVT.SimpleTy != MVT::v16i16)
13904 return 0;
13905 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
13906 return fastEmitInst_rr(X86::VPUNPCKHWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13907 }
13908 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
13909 return fastEmitInst_rr(X86::VPUNPCKHWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13910 }
13911 return 0;
13912}
13913
13914unsigned fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13915 if (RetVT.SimpleTy != MVT::v32i16)
13916 return 0;
13917 if ((Subtarget->hasBWI())) {
13918 return fastEmitInst_rr(X86::VPUNPCKHWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13919 }
13920 return 0;
13921}
13922
13923unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13924 if (RetVT.SimpleTy != MVT::v4i32)
13925 return 0;
13926 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13927 return fastEmitInst_rr(X86::VPUNPCKHDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13928 }
13929 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13930 return fastEmitInst_rr(X86::PUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13931 }
13932 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13933 return fastEmitInst_rr(X86::VPUNPCKHDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13934 }
13935 return 0;
13936}
13937
13938unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13939 if (RetVT.SimpleTy != MVT::v8i32)
13940 return 0;
13941 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13942 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13943 }
13944 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13945 return fastEmitInst_rr(X86::VPUNPCKHDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13946 }
13947 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13948 return fastEmitInst_rr(X86::VPUNPCKHDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13949 }
13950 return 0;
13951}
13952
13953unsigned fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13954 if (RetVT.SimpleTy != MVT::v16i32)
13955 return 0;
13956 if ((Subtarget->hasAVX512())) {
13957 return fastEmitInst_rr(X86::VPUNPCKHDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13958 }
13959 return 0;
13960}
13961
13962unsigned fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13963 if (RetVT.SimpleTy != MVT::v2i64)
13964 return 0;
13965 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13966 return fastEmitInst_rr(X86::VPUNPCKHQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13967 }
13968 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
13969 return fastEmitInst_rr(X86::PUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13970 }
13971 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
13972 return fastEmitInst_rr(X86::VPUNPCKHQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13973 }
13974 return 0;
13975}
13976
13977unsigned fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13978 if (RetVT.SimpleTy != MVT::v4i64)
13979 return 0;
13980 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
13981 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13982 }
13983 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
13984 return fastEmitInst_rr(X86::VPUNPCKHQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13985 }
13986 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
13987 return fastEmitInst_rr(X86::VPUNPCKHQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13988 }
13989 return 0;
13990}
13991
13992unsigned fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
13993 if (RetVT.SimpleTy != MVT::v8i64)
13994 return 0;
13995 if ((Subtarget->hasAVX512())) {
13996 return fastEmitInst_rr(X86::VPUNPCKHQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
13997 }
13998 return 0;
13999}
14000
14001unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14002 if (RetVT.SimpleTy != MVT::v4f32)
14003 return 0;
14004 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14005 return fastEmitInst_rr(X86::VUNPCKHPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14006 }
14007 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14008 return fastEmitInst_rr(X86::UNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14009 }
14010 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14011 return fastEmitInst_rr(X86::VUNPCKHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14012 }
14013 return 0;
14014}
14015
14016unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14017 if (RetVT.SimpleTy != MVT::v8f32)
14018 return 0;
14019 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14020 return fastEmitInst_rr(X86::VUNPCKHPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14021 }
14022 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14023 return fastEmitInst_rr(X86::VUNPCKHPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14024 }
14025 return 0;
14026}
14027
14028unsigned fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14029 if (RetVT.SimpleTy != MVT::v16f32)
14030 return 0;
14031 if ((Subtarget->hasAVX512())) {
14032 return fastEmitInst_rr(X86::VUNPCKHPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14033 }
14034 return 0;
14035}
14036
14037unsigned fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14038 if (RetVT.SimpleTy != MVT::v2f64)
14039 return 0;
14040 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14041 return fastEmitInst_rr(X86::VUNPCKHPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14042 }
14043 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14044 return fastEmitInst_rr(X86::UNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14045 }
14046 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14047 return fastEmitInst_rr(X86::VUNPCKHPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14048 }
14049 return 0;
14050}
14051
14052unsigned fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14053 if (RetVT.SimpleTy != MVT::v4f64)
14054 return 0;
14055 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14056 return fastEmitInst_rr(X86::VUNPCKHPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14057 }
14058 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14059 return fastEmitInst_rr(X86::VUNPCKHPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14060 }
14061 return 0;
14062}
14063
14064unsigned fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14065 if (RetVT.SimpleTy != MVT::v8f64)
14066 return 0;
14067 if ((Subtarget->hasAVX512())) {
14068 return fastEmitInst_rr(X86::VUNPCKHPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14069 }
14070 return 0;
14071}
14072
14073unsigned fastEmit_X86ISD_UNPCKH_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14074 switch (VT.SimpleTy) {
14075 case MVT::v16i8: return fastEmit_X86ISD_UNPCKH_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14076 case MVT::v32i8: return fastEmit_X86ISD_UNPCKH_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14077 case MVT::v64i8: return fastEmit_X86ISD_UNPCKH_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14078 case MVT::v8i16: return fastEmit_X86ISD_UNPCKH_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14079 case MVT::v16i16: return fastEmit_X86ISD_UNPCKH_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14080 case MVT::v32i16: return fastEmit_X86ISD_UNPCKH_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14081 case MVT::v4i32: return fastEmit_X86ISD_UNPCKH_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14082 case MVT::v8i32: return fastEmit_X86ISD_UNPCKH_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14083 case MVT::v16i32: return fastEmit_X86ISD_UNPCKH_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14084 case MVT::v2i64: return fastEmit_X86ISD_UNPCKH_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14085 case MVT::v4i64: return fastEmit_X86ISD_UNPCKH_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14086 case MVT::v8i64: return fastEmit_X86ISD_UNPCKH_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14087 case MVT::v4f32: return fastEmit_X86ISD_UNPCKH_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14088 case MVT::v8f32: return fastEmit_X86ISD_UNPCKH_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14089 case MVT::v16f32: return fastEmit_X86ISD_UNPCKH_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14090 case MVT::v2f64: return fastEmit_X86ISD_UNPCKH_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14091 case MVT::v4f64: return fastEmit_X86ISD_UNPCKH_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14092 case MVT::v8f64: return fastEmit_X86ISD_UNPCKH_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14093 default: return 0;
14094 }
14095}
14096
14097// FastEmit functions for X86ISD::UNPCKL.
14098
14099unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14100 if (RetVT.SimpleTy != MVT::v16i8)
14101 return 0;
14102 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14103 return fastEmitInst_rr(X86::VPUNPCKLBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14104 }
14105 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14106 return fastEmitInst_rr(X86::PUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14107 }
14108 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14109 return fastEmitInst_rr(X86::VPUNPCKLBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14110 }
14111 return 0;
14112}
14113
14114unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14115 if (RetVT.SimpleTy != MVT::v32i8)
14116 return 0;
14117 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14118 return fastEmitInst_rr(X86::VPUNPCKLBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14119 }
14120 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14121 return fastEmitInst_rr(X86::VPUNPCKLBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14122 }
14123 return 0;
14124}
14125
14126unsigned fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14127 if (RetVT.SimpleTy != MVT::v64i8)
14128 return 0;
14129 if ((Subtarget->hasBWI())) {
14130 return fastEmitInst_rr(X86::VPUNPCKLBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14131 }
14132 return 0;
14133}
14134
14135unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14136 if (RetVT.SimpleTy != MVT::v8i16)
14137 return 0;
14138 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14139 return fastEmitInst_rr(X86::VPUNPCKLWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14140 }
14141 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14142 return fastEmitInst_rr(X86::PUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14143 }
14144 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14145 return fastEmitInst_rr(X86::VPUNPCKLWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14146 }
14147 return 0;
14148}
14149
14150unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14151 if (RetVT.SimpleTy != MVT::v16i16)
14152 return 0;
14153 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14154 return fastEmitInst_rr(X86::VPUNPCKLWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14155 }
14156 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14157 return fastEmitInst_rr(X86::VPUNPCKLWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14158 }
14159 return 0;
14160}
14161
14162unsigned fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14163 if (RetVT.SimpleTy != MVT::v32i16)
14164 return 0;
14165 if ((Subtarget->hasBWI())) {
14166 return fastEmitInst_rr(X86::VPUNPCKLWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14167 }
14168 return 0;
14169}
14170
14171unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14172 if (RetVT.SimpleTy != MVT::v4i32)
14173 return 0;
14174 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14175 return fastEmitInst_rr(X86::VPUNPCKLDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14176 }
14177 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14178 return fastEmitInst_rr(X86::PUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14179 }
14180 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14181 return fastEmitInst_rr(X86::VPUNPCKLDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14182 }
14183 return 0;
14184}
14185
14186unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14187 if (RetVT.SimpleTy != MVT::v8i32)
14188 return 0;
14189 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14190 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14191 }
14192 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14193 return fastEmitInst_rr(X86::VPUNPCKLDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14194 }
14195 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14196 return fastEmitInst_rr(X86::VPUNPCKLDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14197 }
14198 return 0;
14199}
14200
14201unsigned fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14202 if (RetVT.SimpleTy != MVT::v16i32)
14203 return 0;
14204 if ((Subtarget->hasAVX512())) {
14205 return fastEmitInst_rr(X86::VPUNPCKLDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14206 }
14207 return 0;
14208}
14209
14210unsigned fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14211 if (RetVT.SimpleTy != MVT::v2i64)
14212 return 0;
14213 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14214 return fastEmitInst_rr(X86::VPUNPCKLQDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14215 }
14216 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14217 return fastEmitInst_rr(X86::PUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14218 }
14219 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14220 return fastEmitInst_rr(X86::VPUNPCKLQDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14221 }
14222 return 0;
14223}
14224
14225unsigned fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14226 if (RetVT.SimpleTy != MVT::v4i64)
14227 return 0;
14228 if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
14229 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14230 }
14231 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14232 return fastEmitInst_rr(X86::VPUNPCKLQDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14233 }
14234 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14235 return fastEmitInst_rr(X86::VPUNPCKLQDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14236 }
14237 return 0;
14238}
14239
14240unsigned fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14241 if (RetVT.SimpleTy != MVT::v8i64)
14242 return 0;
14243 if ((Subtarget->hasAVX512())) {
14244 return fastEmitInst_rr(X86::VPUNPCKLQDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14245 }
14246 return 0;
14247}
14248
14249unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14250 if (RetVT.SimpleTy != MVT::v4f32)
14251 return 0;
14252 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14253 return fastEmitInst_rr(X86::VUNPCKLPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14254 }
14255 if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
14256 return fastEmitInst_rr(X86::UNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14257 }
14258 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14259 return fastEmitInst_rr(X86::VUNPCKLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14260 }
14261 return 0;
14262}
14263
14264unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14265 if (RetVT.SimpleTy != MVT::v8f32)
14266 return 0;
14267 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14268 return fastEmitInst_rr(X86::VUNPCKLPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14269 }
14270 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14271 return fastEmitInst_rr(X86::VUNPCKLPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14272 }
14273 return 0;
14274}
14275
14276unsigned fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14277 if (RetVT.SimpleTy != MVT::v16f32)
14278 return 0;
14279 if ((Subtarget->hasAVX512())) {
14280 return fastEmitInst_rr(X86::VUNPCKLPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14281 }
14282 return 0;
14283}
14284
14285unsigned fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14286 if (RetVT.SimpleTy != MVT::v2f64)
14287 return 0;
14288 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14289 return fastEmitInst_rr(X86::VUNPCKLPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14290 }
14291 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14292 return fastEmitInst_rr(X86::UNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14293 }
14294 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14295 return fastEmitInst_rr(X86::VUNPCKLPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14296 }
14297 return 0;
14298}
14299
14300unsigned fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14301 if (RetVT.SimpleTy != MVT::v4f64)
14302 return 0;
14303 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14304 return fastEmitInst_rr(X86::VUNPCKLPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14305 }
14306 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14307 return fastEmitInst_rr(X86::VUNPCKLPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14308 }
14309 return 0;
14310}
14311
14312unsigned fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14313 if (RetVT.SimpleTy != MVT::v8f64)
14314 return 0;
14315 if ((Subtarget->hasAVX512())) {
14316 return fastEmitInst_rr(X86::VUNPCKLPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14317 }
14318 return 0;
14319}
14320
14321unsigned fastEmit_X86ISD_UNPCKL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14322 switch (VT.SimpleTy) {
14323 case MVT::v16i8: return fastEmit_X86ISD_UNPCKL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14324 case MVT::v32i8: return fastEmit_X86ISD_UNPCKL_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14325 case MVT::v64i8: return fastEmit_X86ISD_UNPCKL_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14326 case MVT::v8i16: return fastEmit_X86ISD_UNPCKL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14327 case MVT::v16i16: return fastEmit_X86ISD_UNPCKL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14328 case MVT::v32i16: return fastEmit_X86ISD_UNPCKL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14329 case MVT::v4i32: return fastEmit_X86ISD_UNPCKL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14330 case MVT::v8i32: return fastEmit_X86ISD_UNPCKL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14331 case MVT::v16i32: return fastEmit_X86ISD_UNPCKL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14332 case MVT::v2i64: return fastEmit_X86ISD_UNPCKL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14333 case MVT::v4i64: return fastEmit_X86ISD_UNPCKL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14334 case MVT::v8i64: return fastEmit_X86ISD_UNPCKL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14335 case MVT::v4f32: return fastEmit_X86ISD_UNPCKL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14336 case MVT::v8f32: return fastEmit_X86ISD_UNPCKL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14337 case MVT::v16f32: return fastEmit_X86ISD_UNPCKL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14338 case MVT::v2f64: return fastEmit_X86ISD_UNPCKL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14339 case MVT::v4f64: return fastEmit_X86ISD_UNPCKL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14340 case MVT::v8f64: return fastEmit_X86ISD_UNPCKL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14341 default: return 0;
14342 }
14343}
14344
14345// FastEmit functions for X86ISD::VP2INTERSECT.
14346
14347unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14348 if (RetVT.SimpleTy != MVT::Untyped)
14349 return 0;
14350 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
14351 return fastEmitInst_rr(X86::VP2INTERSECTDZ128rr, &X86::VK4PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14352 }
14353 return 0;
14354}
14355
14356unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14357 if (RetVT.SimpleTy != MVT::Untyped)
14358 return 0;
14359 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
14360 return fastEmitInst_rr(X86::VP2INTERSECTDZ256rr, &X86::VK8PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14361 }
14362 return 0;
14363}
14364
14365unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14366 if (RetVT.SimpleTy != MVT::Untyped)
14367 return 0;
14368 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
14369 return fastEmitInst_rr(X86::VP2INTERSECTDZrr, &X86::VK16PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14370 }
14371 return 0;
14372}
14373
14374unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14375 if (RetVT.SimpleTy != MVT::Untyped)
14376 return 0;
14377 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
14378 return fastEmitInst_rr(X86::VP2INTERSECTQZ128rr, &X86::VK2PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14379 }
14380 return 0;
14381}
14382
14383unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14384 if (RetVT.SimpleTy != MVT::Untyped)
14385 return 0;
14386 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX()) && (Subtarget->hasVP2INTERSECT())) {
14387 return fastEmitInst_rr(X86::VP2INTERSECTQZ256rr, &X86::VK4PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14388 }
14389 return 0;
14390}
14391
14392unsigned fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14393 if (RetVT.SimpleTy != MVT::Untyped)
14394 return 0;
14395 if ((Subtarget->hasAVX512()) && (Subtarget->hasVP2INTERSECT())) {
14396 return fastEmitInst_rr(X86::VP2INTERSECTQZrr, &X86::VK8PAIRRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14397 }
14398 return 0;
14399}
14400
14401unsigned fastEmit_X86ISD_VP2INTERSECT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14402 switch (VT.SimpleTy) {
14403 case MVT::v4i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14404 case MVT::v8i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14405 case MVT::v16i32: return fastEmit_X86ISD_VP2INTERSECT_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14406 case MVT::v2i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14407 case MVT::v4i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14408 case MVT::v8i64: return fastEmit_X86ISD_VP2INTERSECT_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14409 default: return 0;
14410 }
14411}
14412
14413// FastEmit functions for X86ISD::VPERMV.
14414
14415unsigned fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14416 if (RetVT.SimpleTy != MVT::v16i8)
14417 return 0;
14418 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
14419 return fastEmitInst_rr(X86::VPERMBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14420 }
14421 return 0;
14422}
14423
14424unsigned fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14425 if (RetVT.SimpleTy != MVT::v32i8)
14426 return 0;
14427 if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
14428 return fastEmitInst_rr(X86::VPERMBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14429 }
14430 return 0;
14431}
14432
14433unsigned fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14434 if (RetVT.SimpleTy != MVT::v64i8)
14435 return 0;
14436 if ((Subtarget->hasVBMI())) {
14437 return fastEmitInst_rr(X86::VPERMBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14438 }
14439 return 0;
14440}
14441
14442unsigned fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14443 if (RetVT.SimpleTy != MVT::v8i16)
14444 return 0;
14445 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14446 return fastEmitInst_rr(X86::VPERMWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14447 }
14448 return 0;
14449}
14450
14451unsigned fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14452 if (RetVT.SimpleTy != MVT::v16i16)
14453 return 0;
14454 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14455 return fastEmitInst_rr(X86::VPERMWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14456 }
14457 return 0;
14458}
14459
14460unsigned fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14461 if (RetVT.SimpleTy != MVT::v32i16)
14462 return 0;
14463 if ((Subtarget->hasBWI())) {
14464 return fastEmitInst_rr(X86::VPERMWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14465 }
14466 return 0;
14467}
14468
14469unsigned fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14470 if (RetVT.SimpleTy != MVT::v8i32)
14471 return 0;
14472 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14473 return fastEmitInst_rr(X86::VPERMDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14474 }
14475 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14476 return fastEmitInst_rr(X86::VPERMDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14477 }
14478 return 0;
14479}
14480
14481unsigned fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14482 if (RetVT.SimpleTy != MVT::v16i32)
14483 return 0;
14484 if ((Subtarget->hasAVX512())) {
14485 return fastEmitInst_rr(X86::VPERMDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14486 }
14487 return 0;
14488}
14489
14490unsigned fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14491 if (RetVT.SimpleTy != MVT::v4i64)
14492 return 0;
14493 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14494 return fastEmitInst_rr(X86::VPERMQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14495 }
14496 return 0;
14497}
14498
14499unsigned fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14500 if (RetVT.SimpleTy != MVT::v8i64)
14501 return 0;
14502 if ((Subtarget->hasAVX512())) {
14503 return fastEmitInst_rr(X86::VPERMQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14504 }
14505 return 0;
14506}
14507
14508unsigned fastEmit_X86ISD_VPERMV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14509 switch (VT.SimpleTy) {
14510 case MVT::v16i8: return fastEmit_X86ISD_VPERMV_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14511 case MVT::v32i8: return fastEmit_X86ISD_VPERMV_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14512 case MVT::v64i8: return fastEmit_X86ISD_VPERMV_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14513 case MVT::v8i16: return fastEmit_X86ISD_VPERMV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14514 case MVT::v16i16: return fastEmit_X86ISD_VPERMV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14515 case MVT::v32i16: return fastEmit_X86ISD_VPERMV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14516 case MVT::v8i32: return fastEmit_X86ISD_VPERMV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14517 case MVT::v16i32: return fastEmit_X86ISD_VPERMV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14518 case MVT::v4i64: return fastEmit_X86ISD_VPERMV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14519 case MVT::v8i64: return fastEmit_X86ISD_VPERMV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14520 default: return 0;
14521 }
14522}
14523
14524// FastEmit functions for X86ISD::VPMADDUBSW.
14525
14526unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14527 if (RetVT.SimpleTy != MVT::v8i16)
14528 return 0;
14529 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14530 return fastEmitInst_rr(X86::VPMADDUBSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14531 }
14532 if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
14533 return fastEmitInst_rr(X86::PMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14534 }
14535 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14536 return fastEmitInst_rr(X86::VPMADDUBSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14537 }
14538 return 0;
14539}
14540
14541unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14542 if (RetVT.SimpleTy != MVT::v16i16)
14543 return 0;
14544 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14545 return fastEmitInst_rr(X86::VPMADDUBSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14546 }
14547 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14548 return fastEmitInst_rr(X86::VPMADDUBSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14549 }
14550 return 0;
14551}
14552
14553unsigned fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14554 if (RetVT.SimpleTy != MVT::v32i16)
14555 return 0;
14556 if ((Subtarget->hasBWI())) {
14557 return fastEmitInst_rr(X86::VPMADDUBSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14558 }
14559 return 0;
14560}
14561
14562unsigned fastEmit_X86ISD_VPMADDUBSW_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14563 switch (VT.SimpleTy) {
14564 case MVT::v16i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14565 case MVT::v32i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14566 case MVT::v64i8: return fastEmit_X86ISD_VPMADDUBSW_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14567 default: return 0;
14568 }
14569}
14570
14571// FastEmit functions for X86ISD::VPMADDWD.
14572
14573unsigned fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14574 if (RetVT.SimpleTy != MVT::v4i32)
14575 return 0;
14576 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14577 return fastEmitInst_rr(X86::VPMADDWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14578 }
14579 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14580 return fastEmitInst_rr(X86::PMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14581 }
14582 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14583 return fastEmitInst_rr(X86::VPMADDWDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14584 }
14585 return 0;
14586}
14587
14588unsigned fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14589 if (RetVT.SimpleTy != MVT::v8i32)
14590 return 0;
14591 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14592 return fastEmitInst_rr(X86::VPMADDWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14593 }
14594 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14595 return fastEmitInst_rr(X86::VPMADDWDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14596 }
14597 return 0;
14598}
14599
14600unsigned fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14601 if (RetVT.SimpleTy != MVT::v16i32)
14602 return 0;
14603 if ((Subtarget->hasBWI())) {
14604 return fastEmitInst_rr(X86::VPMADDWDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14605 }
14606 return 0;
14607}
14608
14609unsigned fastEmit_X86ISD_VPMADDWD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14610 switch (VT.SimpleTy) {
14611 case MVT::v8i16: return fastEmit_X86ISD_VPMADDWD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14612 case MVT::v16i16: return fastEmit_X86ISD_VPMADDWD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14613 case MVT::v32i16: return fastEmit_X86ISD_VPMADDWD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14614 default: return 0;
14615 }
14616}
14617
14618// FastEmit functions for X86ISD::VPSHA.
14619
14620unsigned fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14621 if (RetVT.SimpleTy != MVT::v16i8)
14622 return 0;
14623 if ((Subtarget->hasXOP())) {
14624 return fastEmitInst_rr(X86::VPSHABrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14625 }
14626 return 0;
14627}
14628
14629unsigned fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14630 if (RetVT.SimpleTy != MVT::v8i16)
14631 return 0;
14632 if ((Subtarget->hasXOP())) {
14633 return fastEmitInst_rr(X86::VPSHAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14634 }
14635 return 0;
14636}
14637
14638unsigned fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14639 if (RetVT.SimpleTy != MVT::v4i32)
14640 return 0;
14641 if ((Subtarget->hasXOP())) {
14642 return fastEmitInst_rr(X86::VPSHADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14643 }
14644 return 0;
14645}
14646
14647unsigned fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14648 if (RetVT.SimpleTy != MVT::v2i64)
14649 return 0;
14650 if ((Subtarget->hasXOP())) {
14651 return fastEmitInst_rr(X86::VPSHAQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14652 }
14653 return 0;
14654}
14655
14656unsigned fastEmit_X86ISD_VPSHA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14657 switch (VT.SimpleTy) {
14658 case MVT::v16i8: return fastEmit_X86ISD_VPSHA_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14659 case MVT::v8i16: return fastEmit_X86ISD_VPSHA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14660 case MVT::v4i32: return fastEmit_X86ISD_VPSHA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14661 case MVT::v2i64: return fastEmit_X86ISD_VPSHA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14662 default: return 0;
14663 }
14664}
14665
14666// FastEmit functions for X86ISD::VPSHL.
14667
14668unsigned fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14669 if (RetVT.SimpleTy != MVT::v16i8)
14670 return 0;
14671 if ((Subtarget->hasXOP())) {
14672 return fastEmitInst_rr(X86::VPSHLBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14673 }
14674 return 0;
14675}
14676
14677unsigned fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14678 if (RetVT.SimpleTy != MVT::v8i16)
14679 return 0;
14680 if ((Subtarget->hasXOP())) {
14681 return fastEmitInst_rr(X86::VPSHLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14682 }
14683 return 0;
14684}
14685
14686unsigned fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14687 if (RetVT.SimpleTy != MVT::v4i32)
14688 return 0;
14689 if ((Subtarget->hasXOP())) {
14690 return fastEmitInst_rr(X86::VPSHLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14691 }
14692 return 0;
14693}
14694
14695unsigned fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14696 if (RetVT.SimpleTy != MVT::v2i64)
14697 return 0;
14698 if ((Subtarget->hasXOP())) {
14699 return fastEmitInst_rr(X86::VPSHLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14700 }
14701 return 0;
14702}
14703
14704unsigned fastEmit_X86ISD_VPSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14705 switch (VT.SimpleTy) {
14706 case MVT::v16i8: return fastEmit_X86ISD_VPSHL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14707 case MVT::v8i16: return fastEmit_X86ISD_VPSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14708 case MVT::v4i32: return fastEmit_X86ISD_VPSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14709 case MVT::v2i64: return fastEmit_X86ISD_VPSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14710 default: return 0;
14711 }
14712}
14713
14714// FastEmit functions for X86ISD::VPSHUFBITQMB.
14715
14716unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14717 if (RetVT.SimpleTy != MVT::v16i1)
14718 return 0;
14719 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
14720 return fastEmitInst_rr(X86::VPSHUFBITQMBZ128rr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14721 }
14722 return 0;
14723}
14724
14725unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14726 if (RetVT.SimpleTy != MVT::v32i1)
14727 return 0;
14728 if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
14729 return fastEmitInst_rr(X86::VPSHUFBITQMBZ256rr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14730 }
14731 return 0;
14732}
14733
14734unsigned fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14735 if (RetVT.SimpleTy != MVT::v64i1)
14736 return 0;
14737 if ((Subtarget->hasBITALG())) {
14738 return fastEmitInst_rr(X86::VPSHUFBITQMBZrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14739 }
14740 return 0;
14741}
14742
14743unsigned fastEmit_X86ISD_VPSHUFBITQMB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14744 switch (VT.SimpleTy) {
14745 case MVT::v16i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14746 case MVT::v32i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14747 case MVT::v64i8: return fastEmit_X86ISD_VPSHUFBITQMB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14748 default: return 0;
14749 }
14750}
14751
14752// FastEmit functions for X86ISD::VSHL.
14753
14754unsigned fastEmit_X86ISD_VSHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14755 if (RetVT.SimpleTy != MVT::v8i16)
14756 return 0;
14757 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14758 return fastEmitInst_rr(X86::VPSLLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14759 }
14760 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14761 return fastEmitInst_rr(X86::PSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14762 }
14763 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14764 return fastEmitInst_rr(X86::VPSLLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14765 }
14766 return 0;
14767}
14768
14769unsigned fastEmit_X86ISD_VSHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14770 if (RetVT.SimpleTy != MVT::v4i32)
14771 return 0;
14772 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14773 return fastEmitInst_rr(X86::VPSLLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14774 }
14775 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14776 return fastEmitInst_rr(X86::PSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14777 }
14778 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14779 return fastEmitInst_rr(X86::VPSLLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14780 }
14781 return 0;
14782}
14783
14784unsigned fastEmit_X86ISD_VSHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14785 if (RetVT.SimpleTy != MVT::v2i64)
14786 return 0;
14787 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14788 return fastEmitInst_rr(X86::VPSLLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14789 }
14790 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14791 return fastEmitInst_rr(X86::PSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14792 }
14793 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14794 return fastEmitInst_rr(X86::VPSLLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14795 }
14796 return 0;
14797}
14798
14799unsigned fastEmit_X86ISD_VSHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14800 switch (VT.SimpleTy) {
14801 case MVT::v8i16: return fastEmit_X86ISD_VSHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14802 case MVT::v4i32: return fastEmit_X86ISD_VSHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14803 case MVT::v2i64: return fastEmit_X86ISD_VSHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14804 default: return 0;
14805 }
14806}
14807
14808// FastEmit functions for X86ISD::VSHLV.
14809
14810unsigned fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14811 if (RetVT.SimpleTy != MVT::v8i16)
14812 return 0;
14813 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14814 return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14815 }
14816 return 0;
14817}
14818
14819unsigned fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14820 if (RetVT.SimpleTy != MVT::v16i16)
14821 return 0;
14822 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14823 return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14824 }
14825 return 0;
14826}
14827
14828unsigned fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14829 if (RetVT.SimpleTy != MVT::v32i16)
14830 return 0;
14831 if ((Subtarget->hasBWI())) {
14832 return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14833 }
14834 return 0;
14835}
14836
14837unsigned fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14838 if (RetVT.SimpleTy != MVT::v4i32)
14839 return 0;
14840 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14841 return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14842 }
14843 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14844 return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14845 }
14846 return 0;
14847}
14848
14849unsigned fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14850 if (RetVT.SimpleTy != MVT::v8i32)
14851 return 0;
14852 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14853 return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14854 }
14855 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14856 return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14857 }
14858 return 0;
14859}
14860
14861unsigned fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14862 if (RetVT.SimpleTy != MVT::v16i32)
14863 return 0;
14864 if ((Subtarget->hasAVX512())) {
14865 return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14866 }
14867 return 0;
14868}
14869
14870unsigned fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14871 if (RetVT.SimpleTy != MVT::v2i64)
14872 return 0;
14873 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14874 return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14875 }
14876 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14877 return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14878 }
14879 return 0;
14880}
14881
14882unsigned fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14883 if (RetVT.SimpleTy != MVT::v4i64)
14884 return 0;
14885 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14886 return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14887 }
14888 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
14889 return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14890 }
14891 return 0;
14892}
14893
14894unsigned fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14895 if (RetVT.SimpleTy != MVT::v8i64)
14896 return 0;
14897 if ((Subtarget->hasAVX512())) {
14898 return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14899 }
14900 return 0;
14901}
14902
14903unsigned fastEmit_X86ISD_VSHLV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14904 switch (VT.SimpleTy) {
14905 case MVT::v8i16: return fastEmit_X86ISD_VSHLV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14906 case MVT::v16i16: return fastEmit_X86ISD_VSHLV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14907 case MVT::v32i16: return fastEmit_X86ISD_VSHLV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14908 case MVT::v4i32: return fastEmit_X86ISD_VSHLV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14909 case MVT::v8i32: return fastEmit_X86ISD_VSHLV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14910 case MVT::v16i32: return fastEmit_X86ISD_VSHLV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14911 case MVT::v2i64: return fastEmit_X86ISD_VSHLV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14912 case MVT::v4i64: return fastEmit_X86ISD_VSHLV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14913 case MVT::v8i64: return fastEmit_X86ISD_VSHLV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14914 default: return 0;
14915 }
14916}
14917
14918// FastEmit functions for X86ISD::VSRA.
14919
14920unsigned fastEmit_X86ISD_VSRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14921 if (RetVT.SimpleTy != MVT::v8i16)
14922 return 0;
14923 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14924 return fastEmitInst_rr(X86::VPSRAWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14925 }
14926 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14927 return fastEmitInst_rr(X86::PSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14928 }
14929 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
14930 return fastEmitInst_rr(X86::VPSRAWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14931 }
14932 return 0;
14933}
14934
14935unsigned fastEmit_X86ISD_VSRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14936 if (RetVT.SimpleTy != MVT::v4i32)
14937 return 0;
14938 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14939 return fastEmitInst_rr(X86::VPSRADZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14940 }
14941 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
14942 return fastEmitInst_rr(X86::PSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14943 }
14944 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
14945 return fastEmitInst_rr(X86::VPSRADrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14946 }
14947 return 0;
14948}
14949
14950unsigned fastEmit_X86ISD_VSRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14951 if (RetVT.SimpleTy != MVT::v2i64)
14952 return 0;
14953 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
14954 return fastEmitInst_rr(X86::VPSRAQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14955 }
14956 return 0;
14957}
14958
14959unsigned fastEmit_X86ISD_VSRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14960 switch (VT.SimpleTy) {
14961 case MVT::v8i16: return fastEmit_X86ISD_VSRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14962 case MVT::v4i32: return fastEmit_X86ISD_VSRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14963 case MVT::v2i64: return fastEmit_X86ISD_VSRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
14964 default: return 0;
14965 }
14966}
14967
14968// FastEmit functions for X86ISD::VSRAV.
14969
14970unsigned fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14971 if (RetVT.SimpleTy != MVT::v8i16)
14972 return 0;
14973 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14974 return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14975 }
14976 return 0;
14977}
14978
14979unsigned fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14980 if (RetVT.SimpleTy != MVT::v16i16)
14981 return 0;
14982 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
14983 return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14984 }
14985 return 0;
14986}
14987
14988unsigned fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14989 if (RetVT.SimpleTy != MVT::v32i16)
14990 return 0;
14991 if ((Subtarget->hasBWI())) {
14992 return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
14993 }
14994 return 0;
14995}
14996
14997unsigned fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
14998 if (RetVT.SimpleTy != MVT::v4i32)
14999 return 0;
15000 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15001 return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15002 }
15003 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15004 return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15005 }
15006 return 0;
15007}
15008
15009unsigned fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15010 if (RetVT.SimpleTy != MVT::v8i32)
15011 return 0;
15012 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15013 return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15014 }
15015 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15016 return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15017 }
15018 return 0;
15019}
15020
15021unsigned fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15022 if (RetVT.SimpleTy != MVT::v16i32)
15023 return 0;
15024 if ((Subtarget->hasAVX512())) {
15025 return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15026 }
15027 return 0;
15028}
15029
15030unsigned fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15031 if (RetVT.SimpleTy != MVT::v2i64)
15032 return 0;
15033 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15034 return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15035 }
15036 return 0;
15037}
15038
15039unsigned fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15040 if (RetVT.SimpleTy != MVT::v4i64)
15041 return 0;
15042 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15043 return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15044 }
15045 return 0;
15046}
15047
15048unsigned fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15049 if (RetVT.SimpleTy != MVT::v8i64)
15050 return 0;
15051 if ((Subtarget->hasAVX512())) {
15052 return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15053 }
15054 return 0;
15055}
15056
15057unsigned fastEmit_X86ISD_VSRAV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15058 switch (VT.SimpleTy) {
15059 case MVT::v8i16: return fastEmit_X86ISD_VSRAV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15060 case MVT::v16i16: return fastEmit_X86ISD_VSRAV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15061 case MVT::v32i16: return fastEmit_X86ISD_VSRAV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15062 case MVT::v4i32: return fastEmit_X86ISD_VSRAV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15063 case MVT::v8i32: return fastEmit_X86ISD_VSRAV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15064 case MVT::v16i32: return fastEmit_X86ISD_VSRAV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15065 case MVT::v2i64: return fastEmit_X86ISD_VSRAV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15066 case MVT::v4i64: return fastEmit_X86ISD_VSRAV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15067 case MVT::v8i64: return fastEmit_X86ISD_VSRAV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15068 default: return 0;
15069 }
15070}
15071
15072// FastEmit functions for X86ISD::VSRL.
15073
15074unsigned fastEmit_X86ISD_VSRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15075 if (RetVT.SimpleTy != MVT::v8i16)
15076 return 0;
15077 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15078 return fastEmitInst_rr(X86::VPSRLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15079 }
15080 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15081 return fastEmitInst_rr(X86::PSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15082 }
15083 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
15084 return fastEmitInst_rr(X86::VPSRLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15085 }
15086 return 0;
15087}
15088
15089unsigned fastEmit_X86ISD_VSRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15090 if (RetVT.SimpleTy != MVT::v4i32)
15091 return 0;
15092 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15093 return fastEmitInst_rr(X86::VPSRLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15094 }
15095 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15096 return fastEmitInst_rr(X86::PSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15097 }
15098 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15099 return fastEmitInst_rr(X86::VPSRLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15100 }
15101 return 0;
15102}
15103
15104unsigned fastEmit_X86ISD_VSRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15105 if (RetVT.SimpleTy != MVT::v2i64)
15106 return 0;
15107 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15108 return fastEmitInst_rr(X86::VPSRLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15109 }
15110 if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
15111 return fastEmitInst_rr(X86::PSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15112 }
15113 if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
15114 return fastEmitInst_rr(X86::VPSRLQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15115 }
15116 return 0;
15117}
15118
15119unsigned fastEmit_X86ISD_VSRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15120 switch (VT.SimpleTy) {
15121 case MVT::v8i16: return fastEmit_X86ISD_VSRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15122 case MVT::v4i32: return fastEmit_X86ISD_VSRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15123 case MVT::v2i64: return fastEmit_X86ISD_VSRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15124 default: return 0;
15125 }
15126}
15127
15128// FastEmit functions for X86ISD::VSRLV.
15129
15130unsigned fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15131 if (RetVT.SimpleTy != MVT::v8i16)
15132 return 0;
15133 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15134 return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15135 }
15136 return 0;
15137}
15138
15139unsigned fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15140 if (RetVT.SimpleTy != MVT::v16i16)
15141 return 0;
15142 if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
15143 return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15144 }
15145 return 0;
15146}
15147
15148unsigned fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15149 if (RetVT.SimpleTy != MVT::v32i16)
15150 return 0;
15151 if ((Subtarget->hasBWI())) {
15152 return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15153 }
15154 return 0;
15155}
15156
15157unsigned fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15158 if (RetVT.SimpleTy != MVT::v4i32)
15159 return 0;
15160 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15161 return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15162 }
15163 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15164 return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15165 }
15166 return 0;
15167}
15168
15169unsigned fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15170 if (RetVT.SimpleTy != MVT::v8i32)
15171 return 0;
15172 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15173 return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15174 }
15175 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15176 return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15177 }
15178 return 0;
15179}
15180
15181unsigned fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15182 if (RetVT.SimpleTy != MVT::v16i32)
15183 return 0;
15184 if ((Subtarget->hasAVX512())) {
15185 return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15186 }
15187 return 0;
15188}
15189
15190unsigned fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15191 if (RetVT.SimpleTy != MVT::v2i64)
15192 return 0;
15193 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15194 return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15195 }
15196 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15197 return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15198 }
15199 return 0;
15200}
15201
15202unsigned fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15203 if (RetVT.SimpleTy != MVT::v4i64)
15204 return 0;
15205 if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
15206 return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15207 }
15208 if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
15209 return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15210 }
15211 return 0;
15212}
15213
15214unsigned fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15215 if (RetVT.SimpleTy != MVT::v8i64)
15216 return 0;
15217 if ((Subtarget->hasAVX512())) {
15218 return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
15219 }
15220 return 0;
15221}
15222
15223unsigned fastEmit_X86ISD_VSRLV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
15224 switch (VT.SimpleTy) {
15225 case MVT::v8i16: return fastEmit_X86ISD_VSRLV_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15226 case MVT::v16i16: return fastEmit_X86ISD_VSRLV_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15227 case MVT::v32i16: return fastEmit_X86ISD_VSRLV_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15228 case MVT::v4i32: return fastEmit_X86ISD_VSRLV_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15229 case MVT::v8i32: return fastEmit_X86ISD_VSRLV_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15230 case MVT::v16i32: return fastEmit_X86ISD_VSRLV_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15231 case MVT::v2i64: return fastEmit_X86ISD_VSRLV_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15232 case MVT::v4i64: return fastEmit_X86ISD_VSRLV_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15233 case MVT::v8i64: return fastEmit_X86ISD_VSRLV_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15234 default: return 0;
15235 }
15236}
15237
15238// Top-level FastEmit function.
15239
15240unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
15241 switch (Opcode) {
15242 case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15243 case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15244 case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15245 case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15246 case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15247 case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15248 case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15249 case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15250 case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15251 case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15252 case ISD::ROTL: return fastEmit_ISD_ROTL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15253 case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15254 case ISD::SADDSAT: return fastEmit_ISD_SADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15255 case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15256 case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15257 case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15258 case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15259 case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15260 case ISD::SSUBSAT: return fastEmit_ISD_SSUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15261 case ISD::STRICT_FADD: return fastEmit_ISD_STRICT_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15262 case ISD::STRICT_FDIV: return fastEmit_ISD_STRICT_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15263 case ISD::STRICT_FMUL: return fastEmit_ISD_STRICT_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15264 case ISD::STRICT_FSUB: return fastEmit_ISD_STRICT_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15265 case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15266 case ISD::UADDSAT: return fastEmit_ISD_UADDSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15267 case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15268 case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15269 case ISD::USUBSAT: return fastEmit_ISD_USUBSAT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15270 case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15271 case X86ISD::ADDSUB: return fastEmit_X86ISD_ADDSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15272 case X86ISD::ANDNP: return fastEmit_X86ISD_ANDNP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15273 case X86ISD::AVG: return fastEmit_X86ISD_AVG_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15274 case X86ISD::BEXTR: return fastEmit_X86ISD_BEXTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15275 case X86ISD::BT: return fastEmit_X86ISD_BT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15276 case X86ISD::BZHI: return fastEmit_X86ISD_BZHI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15277 case X86ISD::CMP: return fastEmit_X86ISD_CMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15278 case X86ISD::COMI: return fastEmit_X86ISD_COMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15279 case X86ISD::CVTNE2PS2BF16: return fastEmit_X86ISD_CVTNE2PS2BF16_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15280 case X86ISD::FADDS: return fastEmit_X86ISD_FADDS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15281 case X86ISD::FAND: return fastEmit_X86ISD_FAND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15282 case X86ISD::FANDN: return fastEmit_X86ISD_FANDN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15283 case X86ISD::FCMP: return fastEmit_X86ISD_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15284 case X86ISD::FDIVS: return fastEmit_X86ISD_FDIVS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15285 case X86ISD::FGETEXPS: return fastEmit_X86ISD_FGETEXPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15286 case X86ISD::FGETEXPS_SAE: return fastEmit_X86ISD_FGETEXPS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15287 case X86ISD::FHADD: return fastEmit_X86ISD_FHADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15288 case X86ISD::FHSUB: return fastEmit_X86ISD_FHSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15289 case X86ISD::FMAX: return fastEmit_X86ISD_FMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15290 case X86ISD::FMAXC: return fastEmit_X86ISD_FMAXC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15291 case X86ISD::FMAXS: return fastEmit_X86ISD_FMAXS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15292 case X86ISD::FMAXS_SAE: return fastEmit_X86ISD_FMAXS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15293 case X86ISD::FMAX_SAE: return fastEmit_X86ISD_FMAX_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15294 case X86ISD::FMIN: return fastEmit_X86ISD_FMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15295 case X86ISD::FMINC: return fastEmit_X86ISD_FMINC_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15296 case X86ISD::FMINS: return fastEmit_X86ISD_FMINS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15297 case X86ISD::FMINS_SAE: return fastEmit_X86ISD_FMINS_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15298 case X86ISD::FMIN_SAE: return fastEmit_X86ISD_FMIN_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15299 case X86ISD::FMULS: return fastEmit_X86ISD_FMULS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15300 case X86ISD::FOR: return fastEmit_X86ISD_FOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15301 case X86ISD::FSQRTS: return fastEmit_X86ISD_FSQRTS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15302 case X86ISD::FSUBS: return fastEmit_X86ISD_FSUBS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15303 case X86ISD::FXOR: return fastEmit_X86ISD_FXOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15304 case X86ISD::GF2P8MULB: return fastEmit_X86ISD_GF2P8MULB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15305 case X86ISD::HADD: return fastEmit_X86ISD_HADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15306 case X86ISD::HSUB: return fastEmit_X86ISD_HSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15307 case X86ISD::KADD: return fastEmit_X86ISD_KADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15308 case X86ISD::KORTEST: return fastEmit_X86ISD_KORTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15309 case X86ISD::KTEST: return fastEmit_X86ISD_KTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15310 case X86ISD::MOVHLPS: return fastEmit_X86ISD_MOVHLPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15311 case X86ISD::MOVLHPS: return fastEmit_X86ISD_MOVLHPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15312 case X86ISD::MOVSD: return fastEmit_X86ISD_MOVSD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15313 case X86ISD::MOVSS: return fastEmit_X86ISD_MOVSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15314 case X86ISD::MULHRS: return fastEmit_X86ISD_MULHRS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15315 case X86ISD::MULTISHIFT: return fastEmit_X86ISD_MULTISHIFT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15316 case X86ISD::PACKSS: return fastEmit_X86ISD_PACKSS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15317 case X86ISD::PACKUS: return fastEmit_X86ISD_PACKUS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15318 case X86ISD::PCMPEQ: return fastEmit_X86ISD_PCMPEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15319 case X86ISD::PCMPGT: return fastEmit_X86ISD_PCMPGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15320 case X86ISD::PDEP: return fastEmit_X86ISD_PDEP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15321 case X86ISD::PEXT: return fastEmit_X86ISD_PEXT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15322 case X86ISD::PMULDQ: return fastEmit_X86ISD_PMULDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15323 case X86ISD::PMULUDQ: return fastEmit_X86ISD_PMULUDQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15324 case X86ISD::PSADBW: return fastEmit_X86ISD_PSADBW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15325 case X86ISD::PSHUFB: return fastEmit_X86ISD_PSHUFB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15326 case X86ISD::PTEST: return fastEmit_X86ISD_PTEST_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15327 case X86ISD::RCP14S: return fastEmit_X86ISD_RCP14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15328 case X86ISD::RCP28S: return fastEmit_X86ISD_RCP28S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15329 case X86ISD::RCP28S_SAE: return fastEmit_X86ISD_RCP28S_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15330 case X86ISD::RSQRT14S: return fastEmit_X86ISD_RSQRT14S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15331 case X86ISD::RSQRT28S: return fastEmit_X86ISD_RSQRT28S_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15332 case X86ISD::RSQRT28S_SAE: return fastEmit_X86ISD_RSQRT28S_SAE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15333 case X86ISD::SCALEF: return fastEmit_X86ISD_SCALEF_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15334 case X86ISD::SCALEFS: return fastEmit_X86ISD_SCALEFS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15335 case X86ISD::STRICT_FCMP: return fastEmit_X86ISD_STRICT_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15336 case X86ISD::STRICT_FCMPS: return fastEmit_X86ISD_STRICT_FCMPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15337 case X86ISD::TESTP: return fastEmit_X86ISD_TESTP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15338 case X86ISD::UCOMI: return fastEmit_X86ISD_UCOMI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15339 case X86ISD::UNPCKH: return fastEmit_X86ISD_UNPCKH_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15340 case X86ISD::UNPCKL: return fastEmit_X86ISD_UNPCKL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15341 case X86ISD::VP2INTERSECT: return fastEmit_X86ISD_VP2INTERSECT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15342 case X86ISD::VPERMV: return fastEmit_X86ISD_VPERMV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15343 case X86ISD::VPMADDUBSW: return fastEmit_X86ISD_VPMADDUBSW_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15344 case X86ISD::VPMADDWD: return fastEmit_X86ISD_VPMADDWD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15345 case X86ISD::VPSHA: return fastEmit_X86ISD_VPSHA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15346 case X86ISD::VPSHL: return fastEmit_X86ISD_VPSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15347 case X86ISD::VPSHUFBITQMB: return fastEmit_X86ISD_VPSHUFBITQMB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15348 case X86ISD::VSHL: return fastEmit_X86ISD_VSHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15349 case X86ISD::VSHLV: return fastEmit_X86ISD_VSHLV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15350 case X86ISD::VSRA: return fastEmit_X86ISD_VSRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15351 case X86ISD::VSRAV: return fastEmit_X86ISD_VSRAV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15352 case X86ISD::VSRL: return fastEmit_X86ISD_VSRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15353 case X86ISD::VSRLV: return fastEmit_X86ISD_VSRLV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
15354 default: return 0;
15355 }
15356}
15357
15358// FastEmit functions for ISD::ADD.
15359
15360unsigned fastEmit_ISD_ADD_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15361 if (RetVT.SimpleTy != MVT::i8)
15362 return 0;
15363 return fastEmitInst_ri(X86::ADD8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15364}
15365
15366unsigned fastEmit_ISD_ADD_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15367 if (RetVT.SimpleTy != MVT::i16)
15368 return 0;
15369 return fastEmitInst_ri(X86::ADD16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15370}
15371
15372unsigned fastEmit_ISD_ADD_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15373 if (RetVT.SimpleTy != MVT::i32)
15374 return 0;
15375 return fastEmitInst_ri(X86::ADD32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15376}
15377
15378unsigned fastEmit_ISD_ADD_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15379 switch (VT.SimpleTy) {
15380 case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15381 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15382 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15383 default: return 0;
15384 }
15385}
15386
15387// FastEmit functions for ISD::AND.
15388
15389unsigned fastEmit_ISD_AND_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15390 if (RetVT.SimpleTy != MVT::i8)
15391 return 0;
15392 return fastEmitInst_ri(X86::AND8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15393}
15394
15395unsigned fastEmit_ISD_AND_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15396 if (RetVT.SimpleTy != MVT::i16)
15397 return 0;
15398 return fastEmitInst_ri(X86::AND16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15399}
15400
15401unsigned fastEmit_ISD_AND_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15402 if (RetVT.SimpleTy != MVT::i32)
15403 return 0;
15404 return fastEmitInst_ri(X86::AND32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15405}
15406
15407unsigned fastEmit_ISD_AND_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15408 switch (VT.SimpleTy) {
15409 case MVT::i8: return fastEmit_ISD_AND_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15410 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15411 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15412 default: return 0;
15413 }
15414}
15415
15416// FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
15417
15418unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15419 if (RetVT.SimpleTy != MVT::i32)
15420 return 0;
15421 if ((Subtarget->hasDQI())) {
15422 return fastEmitInst_ri(X86::VPEXTRDZrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15423 }
15424 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15425 return fastEmitInst_ri(X86::PEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15426 }
15427 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
15428 return fastEmitInst_ri(X86::VPEXTRDrr, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15429 }
15430 return 0;
15431}
15432
15433unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15434 if (RetVT.SimpleTy != MVT::i64)
15435 return 0;
15436 if ((Subtarget->hasDQI())) {
15437 return fastEmitInst_ri(X86::VPEXTRQZrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15438 }
15439 if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
15440 return fastEmitInst_ri(X86::PEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15441 }
15442 if ((Subtarget->hasAVX()) && (!Subtarget->hasDQI())) {
15443 return fastEmitInst_ri(X86::VPEXTRQrr, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15444 }
15445 return 0;
15446}
15447
15448unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15449 switch (VT.SimpleTy) {
15450 case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri(RetVT, Op0, Op0IsKill, imm1);
15451 case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri(RetVT, Op0, Op0IsKill, imm1);
15452 default: return 0;
15453 }
15454}
15455
15456// FastEmit functions for ISD::MUL.
15457
15458unsigned fastEmit_ISD_MUL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15459 if (RetVT.SimpleTy != MVT::i16)
15460 return 0;
15461 return fastEmitInst_ri(X86::IMUL16rri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15462}
15463
15464unsigned fastEmit_ISD_MUL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15465 if (RetVT.SimpleTy != MVT::i32)
15466 return 0;
15467 return fastEmitInst_ri(X86::IMUL32rri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15468}
15469
15470unsigned fastEmit_ISD_MUL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15471 switch (VT.SimpleTy) {
15472 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15473 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15474 default: return 0;
15475 }
15476}
15477
15478// FastEmit functions for ISD::OR.
15479
15480unsigned fastEmit_ISD_OR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15481 if (RetVT.SimpleTy != MVT::i8)
15482 return 0;
15483 return fastEmitInst_ri(X86::OR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15484}
15485
15486unsigned fastEmit_ISD_OR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15487 if (RetVT.SimpleTy != MVT::i16)
15488 return 0;
15489 return fastEmitInst_ri(X86::OR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15490}
15491
15492unsigned fastEmit_ISD_OR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15493 if (RetVT.SimpleTy != MVT::i32)
15494 return 0;
15495 return fastEmitInst_ri(X86::OR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15496}
15497
15498unsigned fastEmit_ISD_OR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15499 switch (VT.SimpleTy) {
15500 case MVT::i8: return fastEmit_ISD_OR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15501 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15502 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15503 default: return 0;
15504 }
15505}
15506
15507// FastEmit functions for ISD::ROTL.
15508
15509unsigned fastEmit_ISD_ROTL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15510 if (RetVT.SimpleTy != MVT::i8)
15511 return 0;
15512 return fastEmitInst_ri(X86::ROL8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15513}
15514
15515unsigned fastEmit_ISD_ROTL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15516 if (RetVT.SimpleTy != MVT::i16)
15517 return 0;
15518 return fastEmitInst_ri(X86::ROL16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15519}
15520
15521unsigned fastEmit_ISD_ROTL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15522 if (RetVT.SimpleTy != MVT::i32)
15523 return 0;
15524 if ((Subtarget->hasFastSHLDRotate())) {
15525 return fastEmitInst_ri(X86::SHLDROT32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15526 }
15527 return fastEmitInst_ri(X86::ROL32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15528}
15529
15530unsigned fastEmit_ISD_ROTL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15531 if (RetVT.SimpleTy != MVT::i64)
15532 return 0;
15533 if ((Subtarget->hasFastSHLDRotate())) {
15534 return fastEmitInst_ri(X86::SHLDROT64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15535 }
15536 return fastEmitInst_ri(X86::ROL64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15537}
15538
15539unsigned fastEmit_ISD_ROTL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15540 switch (VT.SimpleTy) {
15541 case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15542 case MVT::i16: return fastEmit_ISD_ROTL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15543 case MVT::i32: return fastEmit_ISD_ROTL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15544 case MVT::i64: return fastEmit_ISD_ROTL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15545 default: return 0;
15546 }
15547}
15548
15549// FastEmit functions for ISD::ROTR.
15550
15551unsigned fastEmit_ISD_ROTR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15552 if (RetVT.SimpleTy != MVT::i8)
15553 return 0;
15554 return fastEmitInst_ri(X86::ROR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15555}
15556
15557unsigned fastEmit_ISD_ROTR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15558 if (RetVT.SimpleTy != MVT::i16)
15559 return 0;
15560 return fastEmitInst_ri(X86::ROR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15561}
15562
15563unsigned fastEmit_ISD_ROTR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15564 if (RetVT.SimpleTy != MVT::i32)
15565 return 0;
15566 if ((Subtarget->hasBMI2())) {
15567 return fastEmitInst_ri(X86::RORX32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15568 }
15569 if ((Subtarget->hasFastSHLDRotate())) {
15570 return fastEmitInst_ri(X86::SHRDROT32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15571 }
15572 return fastEmitInst_ri(X86::ROR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15573}
15574
15575unsigned fastEmit_ISD_ROTR_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15576 if (RetVT.SimpleTy != MVT::i64)
15577 return 0;
15578 if ((Subtarget->hasBMI2())) {
15579 return fastEmitInst_ri(X86::RORX64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15580 }
15581 if ((Subtarget->hasFastSHLDRotate())) {
15582 return fastEmitInst_ri(X86::SHRDROT64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15583 }
15584 return fastEmitInst_ri(X86::ROR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15585}
15586
15587unsigned fastEmit_ISD_ROTR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15588 switch (VT.SimpleTy) {
15589 case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15590 case MVT::i16: return fastEmit_ISD_ROTR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15591 case MVT::i32: return fastEmit_ISD_ROTR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15592 case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15593 default: return 0;
15594 }
15595}
15596
15597// FastEmit functions for ISD::SHL.
15598
15599unsigned fastEmit_ISD_SHL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15600 if (RetVT.SimpleTy != MVT::i8)
15601 return 0;
15602 return fastEmitInst_ri(X86::SHL8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15603}
15604
15605unsigned fastEmit_ISD_SHL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15606 if (RetVT.SimpleTy != MVT::i16)
15607 return 0;
15608 return fastEmitInst_ri(X86::SHL16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15609}
15610
15611unsigned fastEmit_ISD_SHL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15612 if (RetVT.SimpleTy != MVT::i32)
15613 return 0;
15614 return fastEmitInst_ri(X86::SHL32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15615}
15616
15617unsigned fastEmit_ISD_SHL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15618 if (RetVT.SimpleTy != MVT::i64)
15619 return 0;
15620 return fastEmitInst_ri(X86::SHL64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15621}
15622
15623unsigned fastEmit_ISD_SHL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15624 switch (VT.SimpleTy) {
15625 case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15626 case MVT::i16: return fastEmit_ISD_SHL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15627 case MVT::i32: return fastEmit_ISD_SHL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15628 case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15629 default: return 0;
15630 }
15631}
15632
15633// FastEmit functions for ISD::SRA.
15634
15635unsigned fastEmit_ISD_SRA_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15636 if (RetVT.SimpleTy != MVT::i8)
15637 return 0;
15638 return fastEmitInst_ri(X86::SAR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15639}
15640
15641unsigned fastEmit_ISD_SRA_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15642 if (RetVT.SimpleTy != MVT::i16)
15643 return 0;
15644 return fastEmitInst_ri(X86::SAR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15645}
15646
15647unsigned fastEmit_ISD_SRA_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15648 if (RetVT.SimpleTy != MVT::i32)
15649 return 0;
15650 return fastEmitInst_ri(X86::SAR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15651}
15652
15653unsigned fastEmit_ISD_SRA_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15654 if (RetVT.SimpleTy != MVT::i64)
15655 return 0;
15656 return fastEmitInst_ri(X86::SAR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15657}
15658
15659unsigned fastEmit_ISD_SRA_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15660 switch (VT.SimpleTy) {
15661 case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15662 case MVT::i16: return fastEmit_ISD_SRA_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15663 case MVT::i32: return fastEmit_ISD_SRA_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15664 case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15665 default: return 0;
15666 }
15667}
15668
15669// FastEmit functions for ISD::SRL.
15670
15671unsigned fastEmit_ISD_SRL_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15672 if (RetVT.SimpleTy != MVT::i8)
15673 return 0;
15674 return fastEmitInst_ri(X86::SHR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15675}
15676
15677unsigned fastEmit_ISD_SRL_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15678 if (RetVT.SimpleTy != MVT::i16)
15679 return 0;
15680 return fastEmitInst_ri(X86::SHR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15681}
15682
15683unsigned fastEmit_ISD_SRL_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15684 if (RetVT.SimpleTy != MVT::i32)
15685 return 0;
15686 return fastEmitInst_ri(X86::SHR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15687}
15688
15689unsigned fastEmit_ISD_SRL_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15690 if (RetVT.SimpleTy != MVT::i64)
15691 return 0;
15692 return fastEmitInst_ri(X86::SHR64ri, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15693}
15694
15695unsigned fastEmit_ISD_SRL_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15696 switch (VT.SimpleTy) {
15697 case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15698 case MVT::i16: return fastEmit_ISD_SRL_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15699 case MVT::i32: return fastEmit_ISD_SRL_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15700 case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15701 default: return 0;
15702 }
15703}
15704
15705// FastEmit functions for ISD::SUB.
15706
15707unsigned fastEmit_ISD_SUB_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15708 if (RetVT.SimpleTy != MVT::i8)
15709 return 0;
15710 return fastEmitInst_ri(X86::SUB8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15711}
15712
15713unsigned fastEmit_ISD_SUB_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15714 if (RetVT.SimpleTy != MVT::i16)
15715 return 0;
15716 return fastEmitInst_ri(X86::SUB16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15717}
15718
15719unsigned fastEmit_ISD_SUB_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15720 if (RetVT.SimpleTy != MVT::i32)
15721 return 0;
15722 return fastEmitInst_ri(X86::SUB32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15723}
15724
15725unsigned fastEmit_ISD_SUB_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15726 switch (VT.SimpleTy) {
15727 case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15728 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15729 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15730 default: return 0;
15731 }
15732}
15733
15734// FastEmit functions for ISD::XOR.
15735
15736unsigned fastEmit_ISD_XOR_MVT_i8_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15737 if (RetVT.SimpleTy != MVT::i8)
15738 return 0;
15739 return fastEmitInst_ri(X86::XOR8ri, &X86::GR8RegClass, Op0, Op0IsKill, imm1);
15740}
15741
15742unsigned fastEmit_ISD_XOR_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15743 if (RetVT.SimpleTy != MVT::i16)
15744 return 0;
15745 return fastEmitInst_ri(X86::XOR16ri, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15746}
15747
15748unsigned fastEmit_ISD_XOR_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15749 if (RetVT.SimpleTy != MVT::i32)
15750 return 0;
15751 return fastEmitInst_ri(X86::XOR32ri, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15752}
15753
15754unsigned fastEmit_ISD_XOR_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15755 switch (VT.SimpleTy) {
15756 case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_ri(RetVT, Op0, Op0IsKill, imm1);
15757 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15758 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15759 default: return 0;
15760 }
15761}
15762
15763// FastEmit functions for X86ISD::BT.
15764
15765unsigned fastEmit_X86ISD_BT_MVT_i16_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15766 if (RetVT.SimpleTy != MVT::i32)
15767 return 0;
15768 return fastEmitInst_ri(X86::BT16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15769}
15770
15771unsigned fastEmit_X86ISD_BT_MVT_i32_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15772 if (RetVT.SimpleTy != MVT::i32)
15773 return 0;
15774 return fastEmitInst_ri(X86::BT32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15775}
15776
15777unsigned fastEmit_X86ISD_BT_MVT_i64_ri(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15778 if (RetVT.SimpleTy != MVT::i32)
15779 return 0;
15780 return fastEmitInst_ri(X86::BT64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
15781}
15782
15783unsigned fastEmit_X86ISD_BT_ri(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15784 switch (VT.SimpleTy) {
15785 case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_ri(RetVT, Op0, Op0IsKill, imm1);
15786 case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_ri(RetVT, Op0, Op0IsKill, imm1);
15787 case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_ri(RetVT, Op0, Op0IsKill, imm1);
15788 default: return 0;
15789 }
15790}
15791
15792// Top-level FastEmit function.
15793
15794unsigned fastEmit_ri(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) override {
15795 if (VT == MVT::i16 && Predicate_i16immSExt8(imm1))
15796 if (unsigned Reg = fastEmit_ri_Predicate_i16immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
15797 return Reg;
15798
15799 if (VT == MVT::i32 && Predicate_i32immSExt8(imm1))
15800 if (unsigned Reg = fastEmit_ri_Predicate_i32immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
15801 return Reg;
15802
15803 if (VT == MVT::i64 && Predicate_i64immSExt8(imm1))
15804 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt8(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
15805 return Reg;
15806
15807 if (VT == MVT::i64 && Predicate_i64immSExt32(imm1))
15808 if (unsigned Reg = fastEmit_ri_Predicate_i64immSExt32(VT, RetVT, Opcode, Op0, Op0IsKill, imm1))
15809 return Reg;
15810
15811 switch (Opcode) {
15812 case ISD::ADD: return fastEmit_ISD_ADD_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15813 case ISD::AND: return fastEmit_ISD_AND_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15814 case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15815 case ISD::MUL: return fastEmit_ISD_MUL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15816 case ISD::OR: return fastEmit_ISD_OR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15817 case ISD::ROTL: return fastEmit_ISD_ROTL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15818 case ISD::ROTR: return fastEmit_ISD_ROTR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15819 case ISD::SHL: return fastEmit_ISD_SHL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15820 case ISD::SRA: return fastEmit_ISD_SRA_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15821 case ISD::SRL: return fastEmit_ISD_SRL_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15822 case ISD::SUB: return fastEmit_ISD_SUB_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15823 case ISD::XOR: return fastEmit_ISD_XOR_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15824 case X86ISD::BT: return fastEmit_X86ISD_BT_ri(VT, RetVT, Op0, Op0IsKill, imm1);
15825 default: return 0;
15826 }
15827}
15828
15829// FastEmit functions for ISD::ADD.
15830
15831unsigned fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15832 if (RetVT.SimpleTy != MVT::i16)
15833 return 0;
15834 return fastEmitInst_ri(X86::ADD16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15835}
15836
15837unsigned fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15838 switch (VT.SimpleTy) {
15839 case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15840 default: return 0;
15841 }
15842}
15843
15844// FastEmit functions for ISD::AND.
15845
15846unsigned fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15847 if (RetVT.SimpleTy != MVT::i16)
15848 return 0;
15849 return fastEmitInst_ri(X86::AND16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15850}
15851
15852unsigned fastEmit_ISD_AND_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15853 switch (VT.SimpleTy) {
15854 case MVT::i16: return fastEmit_ISD_AND_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15855 default: return 0;
15856 }
15857}
15858
15859// FastEmit functions for ISD::MUL.
15860
15861unsigned fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15862 if (RetVT.SimpleTy != MVT::i16)
15863 return 0;
15864 return fastEmitInst_ri(X86::IMUL16rri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15865}
15866
15867unsigned fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15868 switch (VT.SimpleTy) {
15869 case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15870 default: return 0;
15871 }
15872}
15873
15874// FastEmit functions for ISD::OR.
15875
15876unsigned fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15877 if (RetVT.SimpleTy != MVT::i16)
15878 return 0;
15879 return fastEmitInst_ri(X86::OR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15880}
15881
15882unsigned fastEmit_ISD_OR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15883 switch (VT.SimpleTy) {
15884 case MVT::i16: return fastEmit_ISD_OR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15885 default: return 0;
15886 }
15887}
15888
15889// FastEmit functions for ISD::SUB.
15890
15891unsigned fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15892 if (RetVT.SimpleTy != MVT::i16)
15893 return 0;
15894 return fastEmitInst_ri(X86::SUB16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15895}
15896
15897unsigned fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15898 switch (VT.SimpleTy) {
15899 case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15900 default: return 0;
15901 }
15902}
15903
15904// FastEmit functions for ISD::XOR.
15905
15906unsigned fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15907 if (RetVT.SimpleTy != MVT::i16)
15908 return 0;
15909 return fastEmitInst_ri(X86::XOR16ri8, &X86::GR16RegClass, Op0, Op0IsKill, imm1);
15910}
15911
15912unsigned fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15913 switch (VT.SimpleTy) {
15914 case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_ri_Predicate_i16immSExt8(RetVT, Op0, Op0IsKill, imm1);
15915 default: return 0;
15916 }
15917}
15918
15919// Top-level FastEmit function.
15920
15921unsigned fastEmit_ri_Predicate_i16immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15922 switch (Opcode) {
15923 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15924 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15925 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15926 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15927 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15928 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i16immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
15929 default: return 0;
15930 }
15931}
15932
15933// FastEmit functions for ISD::ADD.
15934
15935unsigned fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15936 if (RetVT.SimpleTy != MVT::i32)
15937 return 0;
15938 return fastEmitInst_ri(X86::ADD32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15939}
15940
15941unsigned fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15942 switch (VT.SimpleTy) {
15943 case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15944 default: return 0;
15945 }
15946}
15947
15948// FastEmit functions for ISD::AND.
15949
15950unsigned fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15951 if (RetVT.SimpleTy != MVT::i32)
15952 return 0;
15953 return fastEmitInst_ri(X86::AND32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15954}
15955
15956unsigned fastEmit_ISD_AND_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15957 switch (VT.SimpleTy) {
15958 case MVT::i32: return fastEmit_ISD_AND_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15959 default: return 0;
15960 }
15961}
15962
15963// FastEmit functions for ISD::MUL.
15964
15965unsigned fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15966 if (RetVT.SimpleTy != MVT::i32)
15967 return 0;
15968 return fastEmitInst_ri(X86::IMUL32rri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15969}
15970
15971unsigned fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15972 switch (VT.SimpleTy) {
15973 case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15974 default: return 0;
15975 }
15976}
15977
15978// FastEmit functions for ISD::OR.
15979
15980unsigned fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15981 if (RetVT.SimpleTy != MVT::i32)
15982 return 0;
15983 return fastEmitInst_ri(X86::OR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15984}
15985
15986unsigned fastEmit_ISD_OR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15987 switch (VT.SimpleTy) {
15988 case MVT::i32: return fastEmit_ISD_OR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
15989 default: return 0;
15990 }
15991}
15992
15993// FastEmit functions for ISD::SUB.
15994
15995unsigned fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
15996 if (RetVT.SimpleTy != MVT::i32)
15997 return 0;
15998 return fastEmitInst_ri(X86::SUB32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
15999}
16000
16001unsigned fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16002 switch (VT.SimpleTy) {
16003 case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
16004 default: return 0;
16005 }
16006}
16007
16008// FastEmit functions for ISD::XOR.
16009
16010unsigned fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16011 if (RetVT.SimpleTy != MVT::i32)
16012 return 0;
16013 return fastEmitInst_ri(X86::XOR32ri8, &X86::GR32RegClass, Op0, Op0IsKill, imm1);
16014}
16015
16016unsigned fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16017 switch (VT.SimpleTy) {
16018 case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_ri_Predicate_i32immSExt8(RetVT, Op0, Op0IsKill, imm1);
16019 default: return 0;
16020 }
16021}
16022
16023// Top-level FastEmit function.
16024
16025unsigned fastEmit_ri_Predicate_i32immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16026 switch (Opcode) {
16027 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16028 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16029 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16030 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16031 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16032 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i32immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16033 default: return 0;
16034 }
16035}
16036
16037// FastEmit functions for ISD::ADD.
16038
16039unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16040 if (RetVT.SimpleTy != MVT::i64)
16041 return 0;
16042 return fastEmitInst_ri(X86::ADD64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16043}
16044
16045unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16046 switch (VT.SimpleTy) {
16047 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16048 default: return 0;
16049 }
16050}
16051
16052// FastEmit functions for ISD::AND.
16053
16054unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16055 if (RetVT.SimpleTy != MVT::i64)
16056 return 0;
16057 return fastEmitInst_ri(X86::AND64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16058}
16059
16060unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16061 switch (VT.SimpleTy) {
16062 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16063 default: return 0;
16064 }
16065}
16066
16067// FastEmit functions for ISD::MUL.
16068
16069unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16070 if (RetVT.SimpleTy != MVT::i64)
16071 return 0;
16072 return fastEmitInst_ri(X86::IMUL64rri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16073}
16074
16075unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16076 switch (VT.SimpleTy) {
16077 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16078 default: return 0;
16079 }
16080}
16081
16082// FastEmit functions for ISD::OR.
16083
16084unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16085 if (RetVT.SimpleTy != MVT::i64)
16086 return 0;
16087 return fastEmitInst_ri(X86::OR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16088}
16089
16090unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16091 switch (VT.SimpleTy) {
16092 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16093 default: return 0;
16094 }
16095}
16096
16097// FastEmit functions for ISD::SUB.
16098
16099unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16100 if (RetVT.SimpleTy != MVT::i64)
16101 return 0;
16102 return fastEmitInst_ri(X86::SUB64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16103}
16104
16105unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16106 switch (VT.SimpleTy) {
16107 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16108 default: return 0;
16109 }
16110}
16111
16112// FastEmit functions for ISD::XOR.
16113
16114unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16115 if (RetVT.SimpleTy != MVT::i64)
16116 return 0;
16117 return fastEmitInst_ri(X86::XOR64ri8, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16118}
16119
16120unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16121 switch (VT.SimpleTy) {
16122 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt8(RetVT, Op0, Op0IsKill, imm1);
16123 default: return 0;
16124 }
16125}
16126
16127// Top-level FastEmit function.
16128
16129unsigned fastEmit_ri_Predicate_i64immSExt8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16130 switch (Opcode) {
16131 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16132 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16133 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16134 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16135 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16136 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt8(VT, RetVT, Op0, Op0IsKill, imm1);
16137 default: return 0;
16138 }
16139}
16140
16141// FastEmit functions for ISD::ADD.
16142
16143unsigned fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16144 if (RetVT.SimpleTy != MVT::i64)
16145 return 0;
16146 return fastEmitInst_ri(X86::ADD64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16147}
16148
16149unsigned fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16150 switch (VT.SimpleTy) {
16151 case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16152 default: return 0;
16153 }
16154}
16155
16156// FastEmit functions for ISD::AND.
16157
16158unsigned fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16159 if (RetVT.SimpleTy != MVT::i64)
16160 return 0;
16161 return fastEmitInst_ri(X86::AND64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16162}
16163
16164unsigned fastEmit_ISD_AND_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16165 switch (VT.SimpleTy) {
16166 case MVT::i64: return fastEmit_ISD_AND_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16167 default: return 0;
16168 }
16169}
16170
16171// FastEmit functions for ISD::MUL.
16172
16173unsigned fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16174 if (RetVT.SimpleTy != MVT::i64)
16175 return 0;
16176 return fastEmitInst_ri(X86::IMUL64rri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16177}
16178
16179unsigned fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16180 switch (VT.SimpleTy) {
16181 case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16182 default: return 0;
16183 }
16184}
16185
16186// FastEmit functions for ISD::OR.
16187
16188unsigned fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16189 if (RetVT.SimpleTy != MVT::i64)
16190 return 0;
16191 return fastEmitInst_ri(X86::OR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16192}
16193
16194unsigned fastEmit_ISD_OR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16195 switch (VT.SimpleTy) {
16196 case MVT::i64: return fastEmit_ISD_OR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16197 default: return 0;
16198 }
16199}
16200
16201// FastEmit functions for ISD::SUB.
16202
16203unsigned fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16204 if (RetVT.SimpleTy != MVT::i64)
16205 return 0;
16206 return fastEmitInst_ri(X86::SUB64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16207}
16208
16209unsigned fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16210 switch (VT.SimpleTy) {
16211 case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16212 default: return 0;
16213 }
16214}
16215
16216// FastEmit functions for ISD::XOR.
16217
16218unsigned fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16219 if (RetVT.SimpleTy != MVT::i64)
16220 return 0;
16221 return fastEmitInst_ri(X86::XOR64ri32, &X86::GR64RegClass, Op0, Op0IsKill, imm1);
16222}
16223
16224unsigned fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16225 switch (VT.SimpleTy) {
16226 case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_ri_Predicate_i64immSExt32(RetVT, Op0, Op0IsKill, imm1);
16227 default: return 0;
16228 }
16229}
16230
16231// Top-level FastEmit function.
16232
16233unsigned fastEmit_ri_Predicate_i64immSExt32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
16234 switch (Opcode) {
16235 case ISD::ADD: return fastEmit_ISD_ADD_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16236 case ISD::AND: return fastEmit_ISD_AND_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16237 case ISD::MUL: return fastEmit_ISD_MUL_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16238 case ISD::OR: return fastEmit_ISD_OR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16239 case ISD::SUB: return fastEmit_ISD_SUB_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16240 case ISD::XOR: return fastEmit_ISD_XOR_ri_Predicate_i64immSExt32(VT, RetVT, Op0, Op0IsKill, imm1);
16241 default: return 0;
16242 }
16243}
16244
16245// FastEmit functions for ISD::Constant.
16246
16247unsigned fastEmit_ISD_Constant_MVT_i8_i(MVT RetVT, uint64_t imm0) {
16248 if (RetVT.SimpleTy != MVT::i8)
16249 return 0;
16250 return fastEmitInst_i(X86::MOV8ri, &X86::GR8RegClass, imm0);
16251}
16252
16253unsigned fastEmit_ISD_Constant_MVT_i16_i(MVT RetVT, uint64_t imm0) {
16254 if (RetVT.SimpleTy != MVT::i16)
16255 return 0;
16256 return fastEmitInst_i(X86::MOV16ri, &X86::GR16RegClass, imm0);
16257}
16258
16259unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
16260 if (RetVT.SimpleTy != MVT::i32)
16261 return 0;
16262 return fastEmitInst_i(X86::MOV32ri, &X86::GR32RegClass, imm0);
16263}
16264
16265unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
16266 if (RetVT.SimpleTy != MVT::i64)
16267 return 0;
16268 return fastEmitInst_i(X86::MOV64ri, &X86::GR64RegClass, imm0);
16269}
16270
16271unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
16272 switch (VT.SimpleTy) {
16273 case MVT::i8: return fastEmit_ISD_Constant_MVT_i8_i(RetVT, imm0);
16274 case MVT::i16: return fastEmit_ISD_Constant_MVT_i16_i(RetVT, imm0);
16275 case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
16276 case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
16277 default: return 0;
16278 }
16279}
16280
16281// Top-level FastEmit function.
16282
16283unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
16284 switch (Opcode) {
16285 case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
16286 default: return 0;
16287 }
16288}
16289
16290